{"data":{"id":"203964","type":"status_page","attributes":{"company_name":"Triple Pat","company_url":"https://triplepat.com","contact_url":"mailto:support@triplepat.com","logo_url":"https://d1lppblt9t2x15.cloudfront.net/logos/81773542ca3e62f5c806506cdc7b116b.png","timezone":"America/Adak","subdomain":"triplepat","custom_domain":"status.triplepat.com","announcement":"Don't forget! The entire column has to be red for our users to experience an outage. As long as at least one server is working, the check-in service should work for all users.","logo_remote_url":null,"aggregate_state":"operational","created_at":"2024-12-17T20:42:57.937Z","updated_at":"2025-04-19T18:57:39.551Z"},"relationships":{"sections":{"data":[{"id":"244133","type":"status_page_section"}]},"resources":{"data":[{"id":"8506019","type":"status_page_resource"},{"id":"8506020","type":"status_page_resource"},{"id":"8506021","type":"status_page_resource"},{"id":"8506022","type":"status_page_resource"},{"id":"8506023","type":"status_page_resource"}]},"status_reports":{"data":[{"id":"519873","type":"status_report"},{"id":"544911","type":"status_report"},{"id":"530727","type":"status_report"},{"id":"518114","type":"status_report"},{"id":"518112","type":"status_report"},{"id":"536573","type":"status_report"},{"id":"545733","type":"status_report"}]}}},"included":[{"id":"244133","type":"status_page_section","attributes":{"name":"Our site and mirrors","position":1}},{"id":"8506019","type":"status_page_resource","attributes":{"status_page_section_id":244133,"resource_id":2866908,"resource_type":"Monitor","public_name":"triplepat.com","explanation":"","position":0,"availability":0.999904,"status":"operational","status_history":[{"day":"2025-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-14","status":"downtime","downtime_duration":376.596309,"maintenance_duration":0},{"day":"2025-04-15","status":"downtime","downtime_duration":362.650837,"maintenance_duration":0},{"day":"2025-04-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-19","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"8506020","type":"status_page_resource","attributes":{"status_page_section_id":244133,"resource_id":2866912,"resource_type":"Monitor","public_name":"a.triplepat.com","explanation":"","position":1,"availability":0.986831,"status":"operational","status_history":[{"day":"2025-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-07","status":"downtime","downtime_duration":720.473448,"maintenance_duration":0},{"day":"2025-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-10","status":"downtime","downtime_duration":179.906617,"maintenance_duration":0},{"day":"2025-02-11","status":"downtime","downtime_duration":178.53907,"maintenance_duration":0},{"day":"2025-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-27","status":"downtime","downtime_duration":85166.486143,"maintenance_duration":0},{"day":"2025-02-28","status":"downtime","downtime_duration":15487.799076,"maintenance_duration":0},{"day":"2025-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-19","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"8506021","type":"status_page_resource","attributes":{"status_page_section_id":244133,"resource_id":2866913,"resource_type":"Monitor","public_name":"b.triplepat.com","explanation":"","position":2,"availability":0.983882,"status":"operational","status_history":[{"day":"2025-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-07","status":"downtime","downtime_duration":2160.327147,"maintenance_duration":0},{"day":"2025-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-11","status":"downtime","downtime_duration":358.675987,"maintenance_duration":0},{"day":"2025-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-29","status":"downtime","downtime_duration":46951.308618,"maintenance_duration":0},{"day":"2025-03-30","status":"downtime","downtime_duration":75039.879797,"maintenance_duration":0},{"day":"2025-03-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-19","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"8506022","type":"status_page_resource","attributes":{"status_page_section_id":244133,"resource_id":2866914,"resource_type":"Monitor","public_name":"c.triplepat.com","explanation":"","position":3,"availability":1.0,"status":"operational","status_history":[{"day":"2025-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-19","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"8506023","type":"status_page_resource","attributes":{"status_page_section_id":244133,"resource_id":2866915,"resource_type":"Monitor","public_name":"d.triplepat.com","explanation":"","position":4,"availability":0.999834,"status":"operational","status_history":[{"day":"2025-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-17","status":"downtime","downtime_duration":869.024349,"maintenance_duration":0},{"day":"2025-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-02-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-19","status":"downtime","downtime_duration":411.505189,"maintenance_duration":0},{"day":"2025-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-03-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-04-19","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"3379036","type":"status_update","attributes":{"message":"a.triplepat.com recovered.","published_at":"2025-02-28T14:18:07.799Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506020","status":"resolved"}]}},{"id":"3379059","type":"status_update","attributes":{"message":"We tried to upgrade the OS and reboot the a server yesterday to see if that was a safe and quick operation. It was not. We delayed in bringing it back online because we haven't launched and were in the middle of a major code change.","published_at":"2025-02-28T14:23:00.000Z","published_at_timezone":"Brussels","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506020","status":"resolved"}]}},{"id":"3374179","type":"status_update","attributes":{"message":"a.triplepat.com went down.","published_at":"2025-02-27T10:20:33.513Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506020","status":"downtime"}]}},{"id":"519873","type":"status_report","attributes":{"title":"a.triplepat.com was down after a OS upgrade + reboot","report_type":"manual","starts_at":"2025-02-27T10:20:33.513Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506020","status":"resolved"}],"aggregate_state":"downtime"},"relationships":{"status_updates":{"data":[{"id":"3379036","type":"status_update"},{"id":"3379059","type":"status_update"},{"id":"3374179","type":"status_update"}]}}},{"id":"3525416","type":"status_update","attributes":{"message":"The root cause is a failed nginx deployment due to what looks like a race condition and/or an overly-picky health check. We are auditing the health checks.\n\nRedeploying the exact same config worked, so it's clear that this failure has something to do with either races or ephemeral machine state.","published_at":"2025-04-14T14:10:00.000Z","published_at_timezone":"UTC","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506019","status":"resolved"}]}},{"id":"3524551","type":"status_update","attributes":{"message":"triplepat.com recovered.","published_at":"2025-04-14T09:47:16.346Z","published_at_timezone":null,"notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8506019","status":"resolved"}]}},{"id":"3524528","type":"status_update","attributes":{"message":"triplepat.com went down.","published_at":"2025-04-14T09:40:59.749Z","published_at_timezone":null,"notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8506019","status":"downtime"}]}},{"id":"544911","type":"status_report","attributes":{"title":"triplepat.com is down","report_type":"manual","starts_at":"2025-04-14T09:40:59.749Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506019","status":"resolved"}],"aggregate_state":"downtime"},"relationships":{"status_updates":{"data":[{"id":"3525416","type":"status_update"},{"id":"3524551","type":"status_update"},{"id":"3524528","type":"status_update"}]}}},{"id":"3441229","type":"status_update","attributes":{"message":"d.triplepat.com went down.","published_at":"2025-03-19T10:33:55.609Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506023","status":"downtime"}]}},{"id":"3441242","type":"status_update","attributes":{"message":"d.triplepat.com recovered.","published_at":"2025-03-19T10:40:47.114Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506023","status":"resolved"}]}},{"id":"3479360","type":"status_update","attributes":{"message":"TILAA has network issues more frequently than we (or they!) would like. This was one of them.  Our every-node-is-a-master-node redundancy system means that everything still managed to work fine, however.","published_at":"2025-03-31T10:14:00.000Z","published_at_timezone":"Brussels","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506023","status":"resolved"}]}},{"id":"530727","type":"status_report","attributes":{"title":"d.triplepat.com is down","report_type":"manual","starts_at":"2025-03-19T10:33:55.609Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506023","status":"resolved"}],"aggregate_state":"downtime"},"relationships":{"status_updates":{"data":[{"id":"3441229","type":"status_update"},{"id":"3441242","type":"status_update"},{"id":"3479360","type":"status_update"}]}}},{"id":"3361594","type":"status_update","attributes":{"message":"[Tilaa cloud](https://www.tilaa.com/) (where the d server is hosted) had some network issues that made [d.triplepat.com](https://d.triplepat.com) inaccessible. It was scheduled maintenance that ended up taking more things down than they expected.\n\nhttps://status.tilaa.com/incidents/6hfj589ys0w1 and https://status.tilaa.com/incidents/4b2nb4tdw7mc are the issues on Tilaa's incident report page. \n\nUnclear what was the root cause here, as our incident started between the two.  But I suspect it had the same root cause as the second link, because the timestamps on the first end around when people would have gone home and the timestamps on the second begin when people would have started work.","published_at":"2025-02-17T12:03:00.000Z","published_at_timezone":"Brussels","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506023","status":"resolved"}]}},{"id":"518114","type":"status_report","attributes":{"title":"d server was unreachable repeatedly","report_type":"manual","starts_at":"2025-02-17T12:03:00.000Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506023","status":"resolved"}],"aggregate_state":"resolved"},"relationships":{"status_updates":{"data":[{"id":"3361594","type":"status_update"}]}}},{"id":"3361584","type":"status_update","attributes":{"message":"[a](https://a.triplepat.com) and [b](https://b.triplepat.com) servers were briefly unavailable, twice! Once on 7 Feb 2025 and a second time on 11 Feb 2025.  The a server even had a third outage on 10 Feb 2025! This did not affect any users for two reasons: \n\n1. every server needs to go down before the check-in service becomes unavailable, and \n2. we're not launched.\n\nThroughout these incidents, the servers [triplepat.com](https://triplepat.com), [c](https://c.triplepat.com), and [d](https://d.triplepat.com) remained stably up.\n\n# Cause\n\nA bad nginx config was (repeatedly) deployed while we were adding SMTP reverse-proxy functionality to it.  Our deployment process broke down, like it should, but then we flailed around a bit and accidentally allowed the deployment process to get to b.triplepat.com too, causing more servers to go down.\n\nWe treated the first occurrence like a one-off, and then when in reoccurred we took structural action.\n\n# Events leading up\n\nAn engineer asked an LLM to help them with nginx configs and it hallucinated bad advice for SMTP reverse-proxying.  Because they needed the advice, they didn't realize that the responses were hallucinations, and pushed the config. (First incident, bad configs were rolled back)\n\nOn second try, they still didn't realize how *large* the hallucination was. It turned out to be not just a wrong directive or two, but an entirely wrong-headed approach.  The LLMs were too eager to please.  This led to the second and third incident.\n\n# Immediate fix\n\nEach time things were fixed by deploying a non-broken config.  A little manual work was done to try and speed the process along by the eager engineer. As one might expect, this ended up making the problem worse. \n\nThe lesson remains: trust the tools, they almost always make things happen as reliably as things can.\n\n# Permanent fix\n\nWe added a step in our deployment process that asks a separate nginx invocation to parse the config before attempting to restart the remote nginx server with the new config.  Deployment now correctly crashes and halts on the attempted deployment of configs that nginx won't parse, and does so before attempting to restart the remote server with the bad config.\n\nWe also took the opportunity to add the same protection to other configured services.","published_at":"2025-02-11T11:56:00.000Z","published_at_timezone":"Brussels","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506020","status":"resolved"},{"status_page_resource_id":"8506021","status":"resolved"}]}},{"id":"518112","type":"status_report","attributes":{"title":"a and b were briefly unavailable (multiple times!)","report_type":"manual","starts_at":"2025-02-11T11:56:00.000Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506020","status":"resolved"},{"status_page_resource_id":"8506021","status":"resolved"}],"aggregate_state":"resolved"},"relationships":{"status_updates":{"data":[{"id":"3361584","type":"status_update"}]}}},{"id":"3474646","type":"status_update","attributes":{"message":"b.triplepat.com went down.","published_at":"2025-03-29T19:57:28.691Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506021","status":"downtime"}]}},{"id":"3479353","type":"status_update","attributes":{"message":"The machine b.triplepat.com became unavailable and eventually rebooted due to [a \"power event\" and corresponding Google Cloud outage in its datacenter](https://status.cloud.google.com/incidents/N3Dw7nbJ7rk7qwrtwh7X).  When it came back up, the containers did not all start successfully at boot-time.\n\nIt looks like the internal DNS for the docker-compose network didn't come-up successfully, which meant that the nginx config could not resolve the internal names for redirects, which meant that nginx refused to start.  (Another point for https://isitdns.com)\n\nThe power event in GCP: not our fault\nThe DNS stuff not working after boot: at least a little bit our fault, and certainly something we might want to work to remediate.\n\nNo user check-ins affected, because all other servers were up and running fine this whole time. No data loss. Resiliency works!\n\nThe only TODO item is to make sure our systems reboot cleanly.","published_at":"2025-03-31T10:12:00.000Z","published_at_timezone":"Brussels","notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506021","status":"resolved"}]}},{"id":"3478650","type":"status_update","attributes":{"message":"b.triplepat.com recovered.","published_at":"2025-03-31T05:50:39.879Z","published_at_timezone":null,"notify_subscribers":false,"affected_resources":[{"status_page_resource_id":"8506021","status":"resolved"}]}},{"id":"536573","type":"status_report","attributes":{"title":"b.triplepat.com is down","report_type":"manual","starts_at":"2025-03-29T19:57:28.691Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506021","status":"resolved"}],"aggregate_state":"downtime"},"relationships":{"status_updates":{"data":[{"id":"3474646","type":"status_update"},{"id":"3479353","type":"status_update"},{"id":"3478650","type":"status_update"}]}}},{"id":"3529613","type":"status_update","attributes":{"message":"triplepat.com went down.","published_at":"2025-04-15T15:31:15.231Z","published_at_timezone":null,"notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8506019","status":"downtime"}]}},{"id":"3529632","type":"status_update","attributes":{"message":"triplepat.com recovered.","published_at":"2025-04-15T15:37:17.882Z","published_at_timezone":null,"notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8506019","status":"resolved"}]}},{"id":"545733","type":"status_report","attributes":{"title":"triplepat.com is down","report_type":"automatic","starts_at":"2025-04-15T15:31:15.231Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8506019","status":"resolved"}],"aggregate_state":"downtime"},"relationships":{"status_updates":{"data":[{"id":"3529613","type":"status_update"},{"id":"3529632","type":"status_update"}]}}}]}