phitoduck/cloudwatch-simulator
2
1import streamlit as st2import pandas as pd3from datetime import time, date4from utils import generate_random_data, evaluate_alarm_state, aggregate_data, re_aggregate_data, downsample5from textwrap import dedent6from matplotlib import pyplot as plt7 8# Constants9TODAYS_DATE = date.today()10 11def main():12 st.title("AWS CloudWatch Simulator")13 st.markdown(dedent("""\14 Monitoring and alerting can be confusing to learn. There is some theory you need to understand first.15 16 This app is an interative tutorial to help you understand how to record metrics describing the performance17 of an app, and build alerts off of them using AWS CloudWatch.18 19 Lets get started! 🎉20 """))21 22 # Initialize session state23 initialize_session_state()24 25 # Section 1 - Generate random data26 st.header("1 - Generate a series of measurements")27 st.markdown(dedent("""\28 Suppose we have a REST API with a ✨very popular✨ `GET /greeting?name=...` endpoint.29 30 Each time someone calls the endpoint, we can record how long it takes to respond, aka the ***response latency***.31 32 Use this form to generate a random dataset of response times.33 """))34 35 generate_data_form()36 37 if not st.session_state.df.empty:38 st.markdown("### Recorded request latencies")39 display_dataframe("Raw timeseries events", st.session_state.df)40 st.scatter_chart(st.session_state.df.set_index("timestamp"))41 42 st.markdown(dedent("""\43 #### 🚚 ➡ ☁️44 We can ship these metrics to a time series database such as AWS CloudWatch in a few ways. 45 """))46 st.warning("In the CloudWatch Metrics database, data points are organized into [Namespaces, Metrics, and Dimensions](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html#Namespace). Think of a Metric as a dedicated table in a database for a single timeseries, e.g. response latency measurements.", icon="💡")47 st.markdown(dedent("""\48 #### Option 1 - AWS SDK (good)49 50 Our application could use the AWS SDK to upload the data points using the AWS CloudWatch endpoints, e.g.51 52 ```python53 import boto354 cloudwatch = boto3.client('cloudwatch')55 cloudwatch.put_metric_data(56 Namespace='MyApp',57 MetricData=[58 {59 'MetricName': 'Latency',60 'timestamp': '2021-08-01T12:00:00',61 'Value': 102,62 'Unit': 'Milliseconds'63 },64 ... # more metrics data points, recorded at different times65 ]66 ) 67 ```68 69 It is more cost effective to send data points in a batch, but they can be sent individually as well.70 71 ---72 73 #### Option 2 - Structured Logs (better)74 75 Our application could write metrics to stdout in AWS's [Embedded Metric Format (EMF)](https://www.youtube.com/watch?v=HdopVzW6pX0) (structured JSON) and sent to CloudWatch Logs.76 77 CloudWatch logs automatically extracts metrics from EMF-formatted logs and sends them to CloudWatch Metrics.78 79 That is great because it is 80 81 1. 💰 **cheaper**: you are not charged for calls to CloudWatch's PutMetric endpoint, and 82 2. ⚡️ **faster**: logging to stdout is WAY faster than making a network call--especially a 2-way, synchonous HTTP call. And a side process can batch and send our logs without our app having to slow down or worry about that.83 84 ---85 86 ### Option 3 - Built-in Metrics (best)87 88 Some common metrics, such as API Gateway response latency or Lambda runtime can actually be recorded 89 in CloudWatch Metrics automatically. No code required!90 91 This is ideal, but not all metrics are automatically captured, such as application-specific metrics like "how many OpenAI tokens have we used?"92 93 ---94 """))95 96 if not st.session_state.df.empty:97 98 # Section 2 - Calculate Aggregations99 st.header("2 - AWS aggregates the metrics")100 st.markdown(dedent("""\101 This step represents our metrics data after AWS CloudWatch processes and stores it.102 103 Storing raw metrics data can be expensive 💰 (see [CloudWatch Metrics pricing](https://aws.amazon.com/cloudwatch/pricing/)). If your app has high traffic, or bad code, you could send 100s, 1,000s, or 1,000,000s+ of measurement 104 data points per second to AWS CloudWatch.105 106 This metrics data is meant to be analyzed with queries that power visualizations and alerts--which requires compute--which costs more money the more metrics data you have stored.107 108 AWS CloudWatch generally aggregates data into a ***resolution*** of 5 minute intervals. 109 110 In other words, CloudWatch bins data, genrally into ***periods*** of 5 minutes111 and only stores aggregate statistics for each period. This decreases the amount of data stored and queried by orders of magnitude. ✅112 113 You can pay more for AWS to aggregate data at a "higher" (or "finer") resolution, e.g. 1-minute or even 1-second periods. 114 """))115 st.info("Use this form to aggregate the raw data points into periods of different lengths and plot some of [the many statistics that CloudWatch computes](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Statistics-definitions.html) over aggregated periods.", icon="📌")116 aggregation_form()117 118 if not st.session_state.aggregated_df.empty:119 display_dataframe("Aggregated Statistics over Periods", st.session_state.aggregated_df)120 aggregation_function_input__storage = st.selectbox(121 "Aggregation Statistic (just for exploration; does not affect downstream steps)",122 ['p50', 'p95', 'p99', 'max', 'min', 'average'],123 key='aggregation_function_input__storage',124 help="Select the aggregation function for visualizing the data."125 )126 st.line_chart(st.session_state.aggregated_df.set_index("timestamp")[aggregation_function_input__storage])127 128 # Section 3 - Summary Data Aggregated by Period129 st.header("3 - Optionally aggregate metrics further for Alarms and Dashboards")130 st.markdown(dedent("""\131 You can plot metrics in a CloudWatch dashboard. 132 133 When doing this, you can choose to aggregate the data further or run additional queries on it to analyze it and answer particular questions.134 135 We will skip discussing dashboards and focus on ***alerts*** (or ***alarms*** in CloudWatch terms).136 137 Suppose we want an alert that triggers if our endpoint starts to take longer than usual to respond.138 139 CloudWatch's concept of alarms can alert you when a metric, such as response latency, "breaches" a certain *threshold* for a certain *number of periods*.140 """))141 st.info("Use this form to bin the data into periods (optionally of shorter length than the previous step).\n\nThis will set the period length used to create an alarm in the next step.", icon="📌")142 summary_by_period_form()143 144 if not st.session_state.summary_by_period_df.empty:145 display_dataframe("Summary Data Aggregated by Period (for Alarm)", st.session_state.summary_by_period_df)146 aggregation_function_input__alarm = st.selectbox(147 "Aggregation Statistic (used for alarm evaluation in next step)",148 ['p50', 'p95', 'p99', 'max', 'min', 'average'],149 key='aggregation_function_input__alarm',150 help="Select the aggregation function for visualizing the data."151 )152 st.line_chart(st.session_state.summary_by_period_df.set_index("timestamp")[aggregation_function_input__alarm])153 154 # Section 4 - Evaluate Alarm State155 st.header("4 - Configure and evaluate an alarm")156 157 # define what "breaching" means (threshold and condition) and evaluate the data158 alarm_state_form()159 plot_time_series(st.session_state.summary_by_period_df, st.session_state.threshold_input, st.session_state.alarm_condition_input, st.session_state.evaluation_range_input)160 datapoints_to_alarm_input = st.number_input("Datapoints to Alarm", min_value=1, value=3, key='datapoints_to_alarm_input', help="Specify the number of data points with in the overall evaluation range that must be breaching in order to trigger an alarm.")161 evaluate_breaching_data_points()162 st.write("%d out of %d data points must be breaching to trigger an alarm." % (st.session_state.datapoints_to_alarm_input, st.session_state.evaluation_range_input)) 163 display_alarm_state_evaluation(st.session_state.alarm_state_df)164 165 display_key_tables()166 167def initialize_session_state() -> None:168 if 'df' not in st.session_state:169 st.session_state.df = pd.DataFrame()170 if 'aggregated_df' not in st.session_state:171 st.session_state.aggregated_df = pd.DataFrame()172 if 'summary_by_period_df' not in st.session_state:173 st.session_state.summary_by_period_df = pd.DataFrame()174 if 'alarm_state_df' not in st.session_state:175 st.session_state.alarm_state_df = pd.DataFrame()176 177def generate_data_form() -> None:178 with st.form(key='generate_data_form'):179 start_time_input = st.time_input("Start Time", time(12, 0), help="No generated data points will have earlier timestamps than this.")180 end_time_input = st.time_input("End Time", time(12, 30), help="No generated data points will have later timestamps than this.")181 count_input = st.slider("Number of requests", min_value=1, max_value=200, value=20, help="Specify the number of data points to generate.")182 response_time_range_input = st.slider("Response Time Range (ms)", min_value=50, max_value=300, value=(140, 180), help="Select the range of response times in milliseconds. The generated response latencies will be in this range.")183 null_percentage_input = st.slider("Percentage of null data points", min_value=0.0, max_value=1.0, value=0., help="Select the percentage of null values in the generated data. We will use this to simulate 'missing data'--or time periods where no requests were recorded.\n\nCloudWatch does not actually have a concept of data points with null values.")184 submit_button = st.form_submit_button(label='Generate Data')185 186 if submit_button:187 st.session_state.df = generate_random_data(188 date=TODAYS_DATE,189 start_time=start_time_input,190 end_time=end_time_input,191 count=count_input,192 response_time_range=response_time_range_input,193 null_percentage=null_percentage_input194 )195 196def aggregation_form() -> None:197 freq_input = st.selectbox("Storage resolution for metric", ['1min', '2min', '3min', '5min', '10min', '15min'], key='freq_input', help="Select the frequency for aggregating the data.")198 if not st.session_state.df.empty:199 st.session_state.aggregated_df = aggregate_data(st.session_state.df, freq_input)200 201def summary_by_period_form() -> None:202 period_length_input = st.selectbox("Period Length", ['1min', '2min', '3min', '5min', '10min', '15min'], key='period_length_input', help="Select the period length for aggregating the summary data.")203 if not st.session_state.aggregated_df.empty:204 agg_period = int(st.session_state.freq_input.replace('min', ''))205 new_period = int(period_length_input.replace('min', ''))206 207 if new_period < agg_period:208 st.warning(f"The data from Step 2 was downsampled from a {agg_period}-minute resolution to a {new_period}-minute resolution.\n\nRepresentative values for each finer-resolution period were interpolated.", icon="📌")209 elif new_period > agg_period:210 st.warning(f"The data from Step 2 was re-aggregated to a lower resolution (longer period) of {new_period} minutes.\n\nThe resulting values for min, max, and average reflect the values of the collected metrics, but p50, p95, and p99 are merely estimates.", icon="📌")211 212 if new_period < agg_period:213 st.session_state.summary_by_period_df = downsample(st.session_state.aggregated_df, new_period)214 else:215 st.session_state.summary_by_period_df = re_aggregate_data(st.session_state.aggregated_df, period_length_input)216 217def alarm_state_form() -> None:218 threshold_input = st.slider("Threshold (ms)", min_value=50, max_value=300, value=160, key='threshold_input', help="Specify the threshold value for evaluating the alarm state.")219 alarm_condition_input = st.selectbox(220 "Alarm Condition",221 ['>', '>=', '<', '<='],222 key='alarm_condition_input',223 help="Select the condition for evaluating the alarm state."224 )225 226 evaluation_range_input = st.number_input("Evaluation Range (# periods btw green bars)", min_value=1, value=5, key='evaluation_range_input', help="Specify the number of consecutive data points to evaluate for alarm state.")227 228def evaluate_breaching_data_points() -> None:229 if not st.session_state.summary_by_period_df.empty:230 st.session_state.alarm_state_df = evaluate_alarm_state(231 summary_df=st.session_state.summary_by_period_df,232 threshold=st.session_state.threshold_input,233 datapoints_to_alarm=st.session_state.datapoints_to_alarm_input,234 evaluation_range=st.session_state.evaluation_range_input,235 aggregation_function=st.session_state.aggregation_function_input__alarm,236 alarm_condition=st.session_state.alarm_condition_input237 ) 238 239def display_dataframe(title: str, df: pd.DataFrame) -> None:240 st.write(title)241 st.dataframe(df)242 243def plot_time_series(df: pd.DataFrame, threshold: int, alarm_condition: str, evaluation_range: int) -> None:244 timestamps = df['timestamp']245 response_times = df[st.session_state.aggregation_function_input__alarm]246 247 segments = []248 current_segment = {'timestamps': [], 'values': []}249 250 for timestamp, value in zip(timestamps, response_times):251 if pd.isna(value):252 if current_segment['timestamps']:253 segments.append(current_segment)254 current_segment = {'timestamps': [], 'values': []}255 else:256 current_segment['timestamps'].append(timestamp)257 current_segment['values'].append(value)258 259 if current_segment['timestamps']:260 segments.append(current_segment)261 262 fig, ax1 = plt.subplots()263 264 color = 'tab:blue'265 ax1.set_xlabel('timestamp')266 ax1.set_ylabel('Response Time (ms)', color=color)267 268 for segment in segments:269 ax1.plot(segment['timestamps'], segment['values'], color=color, linewidth=0.5)270 ax1.scatter(segment['timestamps'], segment['values'], color=color, s=10)271 272 line_style = '--' if alarm_condition in ['<', '>'] else '-'273 ax1.axhline(y=threshold, color='r', linestyle=line_style, linewidth=0.8, label='Threshold')274 ax1.tick_params(axis='y', labelcolor=color)275 276 if alarm_condition in ['<=', '<']:277 ax1.fill_between(timestamps, 0, threshold, color='pink', alpha=0.3)278 else:279 ax1.fill_between(timestamps, threshold, response_times.max(), color='pink', alpha=0.3)280 281 period_indices = range(len(df))282 ax2 = ax1.twiny()283 ax2.set_xticks(period_indices)284 ax2.set_xticklabels(period_indices, fontsize=8)285 ax2.set_xlabel('Time Periods', fontsize=8)286 ax2.xaxis.set_tick_params(width=0.5)287 288 for idx in period_indices:289 if idx % evaluation_range == 0:290 ax1.axvline(x=df['timestamp'].iloc[idx], color='green', linestyle='-', alpha=0.3)291 max_value = max(filter(lambda x: x is not None, df[st.session_state.aggregation_function_input__alarm]))292 ax1.text(df['timestamp'].iloc[idx], max_value * 0.95, f"[{idx // evaluation_range}]", rotation=90, verticalalignment='bottom', color='grey', alpha=0.7, fontsize=8)293 else:294 ax1.axvline(x=df['timestamp'].iloc[idx], color='grey', linestyle='--', alpha=0.3)295 296 ax1.annotate('Alarm threshold', xy=(0.98, threshold), xycoords=('axes fraction', 'data'), ha='right', va='bottom', fontsize=8, color='red', backgroundcolor='none')297 298 fig.tight_layout()299 st.pyplot(fig)300 301def display_alarm_state_evaluation(df: pd.DataFrame) -> None:302 st.write("Alarm State Evaluation")303 st.dataframe(df)304 305def display_key_tables() -> None:306 st.write("### Key")307 308 # Symbols309 st.write("#### Symbols")310 symbol_data = {311 "Symbol": ["🔴", "⚫️", "🟢"],312 "Meaning": [313 "Breaching data point: This data point breaches the threshold and alarm condition (<, <=, >=, >)",314 "Missing data point: This data point is missing or not reported",315 "Non-breaching data point: This data point is does not breach the threshold and alarm condition (<, <=, >=, >)"316 ]317 }318 symbol_df = pd.DataFrame(symbol_data)319 st.table(symbol_df)320 321 # Columns322 st.write(dedent("""\323 #### Columns: [The 4 Strategies](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/AlarmThatSendsEmail.html#alarms-and-missing-data) for handling missing data points324 325 Sometimes, no metric events may have been reported during a given time period. In this case,326 you must decide how you will treat missing data points. Ignore it? Or consider it a failure.327 328 Here are the 4 supported strategies in AWS:329 """))330 331 column_data = {332 "Strategy": ["missing", "ignore", "breaching", "notBreaching"],333 "Explanation": [334 "If all data points in the alarm evaluation range are missing, the alarm transitions to INSUFFICIENT_DATA. Possible values: INSUFFICIENT_DATA, Retain current state, ALARM, OK.",335 "The current alarm state is maintained. Possible values: Retain current state, ALARM, OK.",336 "Missing data points are treated as \"bad\" and breaching the threshold. Possible values: ALARM, OK.",337 "Missing data points are treated as \"good\" and within the threshold. Possible values: ALARM, OK."338 ]339 }340 column_df = pd.DataFrame(column_data)341 st.table(column_df)342 343 # States344 st.write("#### States")345 state_data = {346 "State": ["ALARM", "OK", "Retain current state", "INSUFFICIENT_DATA"],347 "Description": [348 "Alarm state is triggered.",349 "Everything is within the threshold.",350 "The current alarm state is maintained.",351 "Not enough data to make a determination."352 ]353 }354 state_df = pd.DataFrame(state_data)355 st.table(state_df)356 357if __name__ == "__main__":358 main()359