// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2016-2020 Datadog, Inc.

package check

import (
	"reflect"
	"sort"
	"testing"
)

func Test_findErrors(t *testing.T) {
	tests := []struct {
		name       string
		statusJSON string
		errors     []string
		found      bool
		wantErr    bool
	}{
		{
			name:       "1 error found",
			statusJSON: oneErrorFound,
			errors:     []string{"cri:permanent failure in criutil: retry number exceeded"},
			found:      true,
			wantErr:    false,
		},
		{
			name:       "2 errors found",
			statusJSON: twoErrorsFound,
			errors:     []string{"cri:permanent failure in criutil: retry number exceeded", "redisdb:766ed21d64724d11:You must specify a host/port couple or a unix_socket_path"},
			found:      true,
			wantErr:    false,
		},
		{
			name:       "empty payload",
			statusJSON: "",
			errors:     nil,
			found:      false,
			wantErr:    true,
		},
		{
			name:       "invalid json",
			statusJSON: invalidPayload,
			errors:     nil,
			found:      false,
			wantErr:    true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, got1, err := findErrors(tt.statusJSON)
			if (err != nil) != tt.wantErr {
				t.Errorf("findErrors() error = %v, wantErr %v", err, tt.wantErr)
				return
			}
			sort.Strings(got)
			if !reflect.DeepEqual(got, tt.errors) {
				t.Errorf("findErrors() got = %v, want %v", got, tt.errors)
			}
			if got1 != tt.found {
				t.Errorf("findErrors() got1 = %v, want %v", got1, tt.found)
			}
		})
	}
}

var (
	oneErrorFound = `{"JMXStatus":{"checks":{"initialized_checks":null,"failed_checks":null},"timestamp":0},"agent_metadata":{"cloud_provider":"GCP","hostname_source":"configuration"},"agent_start":"2020-03-11 13:29:20.045215 UTC","aggregatorStats":{"ChecksHistogramBucketMetricSample":0,"ChecksMetricSample":48889274,"DogstatsdMetricSample":1,"Event":3,"EventsFlushErrors":0,"EventsFlushed":3,"Flush":{"ChecksMetricSampleFlushTime":{"FlushIndex":1,"Flushes":[15005882,22306023,21804661,12585729,13180405,13471728,22009343,21051567,20035640,13059532,12920238,20905621,12816095,21655943,13226249,12667409,19684526,20605875,12764489,30680485,13674638,22576940,20757245,20110094,13169895,24907968,20422039,13818256,23156190,12914053,14109305,12931507],"LastFlush":22306023,"Name":"ChecksMetricSampleFlushTime"},"EventFlushTime":{"FlushIndex":1,"Flushes":[1213542,1327310,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":1327310,"Name":"EventFlushTime"},"MainFlushTime":{"FlushIndex":1,"Flushes":[1612252,93663,87607,81969,81697,91348,97315,89408,88853,78886,82591,91407,98623,102474,93963,85257,90161,83328,85724,85482,92260,79978,92002,94654,92381,96759,90841,94911,88331,89714,80192,107546],"LastFlush":93663,"Name":"MainFlushTime"},"MetricSketchFlushTime":{"FlushIndex":-1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"MetricSketchFlushTime"},"ServiceCheckFlushTime":{"FlushIndex":1,"Flushes":[3741571,1477474,1494696,1488186,1462108,1549005,3024901,1444578,1539708,1251693,1479335,10095245,1617446,1477123,1478274,1534855,5815791,1460151,1500046,9856635,1442813,1493157,5829707,1656610,1496061,8527680,1433064,1670784,1474232,1462695,1466503,1553104],"LastFlush":1477474,"Name":"ServiceCheckFlushTime"}},"FlushCount":{"Events":{"FlushIndex":1,"Flushes":[1,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":2,"Name":"Events"},"Series":{"FlushIndex":1,"Flushes":[1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267,1267],"LastFlush":1267,"Name":"Series"},"ServiceChecks":{"FlushIndex":1,"Flushes":[22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22],"LastFlush":22,"Name":"ServiceChecks"},"Sketches":{"FlushIndex":1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"Sketches"}},"HostnameUpdate":0,"NumberOfFlush":33762,"SeriesFlushErrors":0,"SeriesFlushed":42776122,"ServiceCheck":710131,"ServiceCheckFlushErrors":0,"ServiceCheckFlushed":743885,"SketchesFlushErrors":0,"SketchesFlushed":0},"autoConfigStats":{"ConfigErrors":{},"ResolveWarnings":{"apache":["No service found with this AD identifier: httpd","Can't resolve the template for apache at this moment."],"cilium":["No service found with this AD identifier: cilium-agent","Can't resolve the template for cilium at this moment."],"consul":["No service found with this AD identifier: consul","Can't resolve the template for consul at this moment."],"coredns":["No service found with this AD identifier: coredns","Can't resolve the template for coredns at this moment."],"couch":["No service found with this AD identifier: couchdb","Can't resolve the template for couch at this moment."],"couchbase":["No service found with this AD identifier: couchbase","Can't resolve the template for couchbase at this moment."],"elastic":["No service found with this AD identifier: elasticsearch","No service found with this AD identifier: elasticsearch-oss","Can't resolve the template for elastic at this moment."],"etcd":["No service found with this AD identifier: etcd","Can't resolve the template for etcd at this moment."],"external_dns":["No service found with this AD identifier: external-dns","Can't resolve the template for external_dns at this moment."],"harbor":["No service found with this AD identifier: nginx-photon","Can't resolve the template for harbor at this moment."],"kube_apiserver_metrics":["No service found with this AD identifier: kube-apiserver","Can't resolve the template for kube_apiserver_metrics at this moment."],"kyototycoon":["No service found with this AD identifier: kyototycoon","Can't resolve the template for kyototycoon at this moment."],"mcache":["No service found with this AD identifier: memcached","Can't resolve the template for mcache at this moment."],"presto":["No service found with this AD identifier: presto","Can't resolve the template for presto at this moment."],"redisdb":["No service found with this AD identifier: redis","Can't resolve the template for redisdb at this moment."],"riak":["No service found with this AD identifier: riak","Can't resolve the template for riak at this moment."],"tomcat":["No service found with this AD identifier: tomcat","Can't resolve the template for tomcat at this moment."]}},"build_arch":"amd64","checkSchedulerStats":{"LoaderErrors":{"kubernetes_state":{"Core Check Loader":"Check kubernetes_state not found in Catalog","JMX Check Loader":"check is not a jmx check, or unable to determine if it's so","Python Check Loader":"Could not configure any python check kubernetes_state"}},"RunErrors":{}},"clusterAgentStatus":{"Endpoint":"https://10.3.245.213:5005","Version":"1.5.2+commit.60ee741"},"conf_file":"/etc/datadog-agent/datadog.yaml","config":{"additional_checksd":"/etc/datadog-agent/checks.d","confd_path":"/etc/datadog-agent/conf.d","log_file":"","log_level":"INFO"},"dogstatsdStats":{"EventPackets":0,"EventParseErrors":0,"MetricPackets":0,"MetricParseErrors":0,"ServiceCheckPackets":0,"ServiceCheckParseErrors":0,"UdpBytes":0,"UdpPacketReadingErrors":0,"UdpPackets":1,"UdsBytes":0,"UdsOriginDetectionErrors":0,"UdsPacketReadingErrors":0,"UdsPackets":0},"endpointsInfos":{"https://app.datadoghq.com":["XXXXX"]},"forwarderStats":{"APIKeyStatus":{"API key ending with XXXXX":"API Key valid"},"Transactions":{"CheckRunsV1":33762,"Dropped":0,"DroppedOnInput":0,"Errors":0,"ErrorsByType":{"ConnectionErrors":0,"DNSErrors":0,"SentRequestErrors":0,"TLSErrors":0,"WroteRequestErrors":0},"Events":0,"HTTPErrors":0,"HTTPErrorsByCode":{},"HostMetadata":0,"IntakeV1":3660,"Metadata":0,"Requeued":0,"Retried":0,"RetryQueueSize":0,"Series":0,"ServiceChecks":0,"SketchSeries":0,"Success":71184,"TimeseriesV1":33762}},"go_version":"go1.12.9","hostTags":["cluster_name:cluster-XXXXX","XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-k2sn.c.XXXXX-XXXXX.internal","instance-id:7797502609673530187","project:XXXXX-XXXXX","numeric_project_id:958371799887","google-compute-enable-pcid:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-location:europe-west1-b","cluster-name:XXXXX","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","enable-oslogin:false"],"hostinfo":{"hostname":"datadog-agent-jfvxh","uptime":53762,"bootTime":1583879601,"procs":57,"os":"linux","platform":"debian","platformFamily":"debian","platformVersion":"bullseye/sid","kernelVersion":"4.14.138+","virtualizationSystem":"","virtualizationRole":"","hostid":"277eb388-6222-6bd3-39ab-378f7951c4bd"},"hostnameStats":{"errors":{},"provider":"configuration"},"inventories":{"nginx:My Nginx Endpoints:ebb9d244d587b6a5":{"version.major":"1","version.minor":"17","version.patch":"9","version.raw":"1.17.9","version.scheme":"semver"}},"logsStats":{"is_running":false,"endpoints":null,"metrics":null,"integrations":null,"errors":null,"warnings":null},"metadata":{"os":"linux","python":"3.7.6 (default, Feb 20 2020, 13:26:46) [GCC 4.7.2]","systemStats":{"cpuCores":1,"machine":"amd64","platform":"linux","pythonV":"3.7.6","processor":"Intel(R) Xeon(R) CPU @ 2.30GHz","macV":["","",""],"nixV":["debian","bullseye/sid",""],"fbsdV":["","",""],"winV":["","",""]},"meta":{"socket-hostname":"datadog-agent-jfvxh","timezones":["UTC"],"socket-fqdn":"datadog-agent-jfvxh","ec2-hostname":"","hostname":"XXXXX-4658d5d4-k2sn","host_aliases":["XXXXX-4658d5d4-k2sn.XXXXX-XXXXX","XXXXX-4658d5d4-k2sn-cluster-XXXXX"],"instance-id":""},"host-tags":{"system":["cluster_name:cluster-XXXXX"],"google cloud platform":["XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-k2sn.c.XXXXX-XXXXX.internal","instance-id:7797502609673530187","project:XXXXX-XXXXX","numeric_project_id:958371799887","google-compute-enable-pcid:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-location:europe-west1-b","cluster-name:XXXXX","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","enable-oslogin:false"]},"container-meta":{"docker_swarm":"inactive","docker_version":"18.09.7","kubelet_version":"v1.13.12-XXXXX.25"},"network":{"network-id":"projects/958371799887/networks/default"}},"ntpOffset":-0.000092779,"pid":1,"pyLoaderStats":{"ConfigureErrors":{"kubernetes_state (5.1.0)":["could not invoke 'kubernetes_state' python check constructor. New constructor API returned:\n__init__() missing 1 required positional argument: 'agentConfig'Deprecated constructor API returned:\nTraceback (most recent call last):\n  File \"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/kubernetes_state/kubernetes_state.py\", line 76, in __init__\n    kubernetes_state_instance = self._create_kubernetes_state_prometheus_instance(instance)\n  File \"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/kubernetes_state/kubernetes_state.py\", line 176, in _create_kubernetes_state_prometheus_instance\n    raise CheckException(\"Unable to find kube_state_url in config file.\")\ndatadog_checks.base.errors.CheckException: Unable to find kube_state_url in config file.\n"]},"Py3Warnings":{}},"pythonInit":{"Errors":[]},"python_version":"3.7.6","runnerStats":{"Checks":{"cpu":{"cpu":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cpu.d/conf.yaml.default","CheckID":"cpu","CheckName":"cpu","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202566,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439786}},"cri":{"cri":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cri.d/conf.yaml.default","CheckID":"cri","CheckName":"cri","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"permanent failure in criutil: retry number exceeded","LastExecutionTime":0,"LastWarnings":["Error initialising check: permanent failure in criutil: retry number exceeded"],"MetricSamples":0,"ServiceChecks":0,"TotalErrors":33762,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":33762,"UpdateTimestamp":1584439793}},"disk":{"disk:e5dffb8bef24336f":{"AverageExecutionTime":22,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/disk.d/conf.yaml.default","CheckID":"disk:e5dffb8bef24336f","CheckName":"disk","CheckVersion":"2.6.0","Events":0,"ExecutionTimes":[22,22,22,22,22,22,23,22,21,30,22,23,22,21,23,23,22,21,29,21,22,22,21,23,23,23,22,22,21,21,21,21],"LastError":"","LastExecutionTime":22,"LastWarnings":[],"MetricSamples":228,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000008,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439785}},"docker":{"docker":{"AverageExecutionTime":48,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/docker.d/conf.yaml.default","CheckID":"docker","CheckName":"docker","CheckVersion":"","Events":0,"ExecutionTimes":[52,40,39,50,51,50,41,68,38,66,68,82,52,47,49,50,41,38,49,48,40,49,42,38,50,50,38,53,39,38,38,48],"LastError":"","LastExecutionTime":40,"LastWarnings":[],"MetricSamples":370,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":2,"TotalMetricSamples":1000110,"TotalRuns":33762,"TotalServiceChecks":33762,"TotalWarnings":0,"UpdateTimestamp":1584439792}},"file_handle":{"file_handle":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/file_handle.d/conf.yaml.default","CheckID":"file_handle","CheckName":"file_handle","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":5,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":168810,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439784}},"io":{"io":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/io.d/conf.yaml.default","CheckID":"io","CheckName":"io","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":182,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000146,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439791}},"kube_dns":{"kube_dns:4ea3f98a50351efb":{"AverageExecutionTime":14,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/kube_dns.d/auto_conf.yaml","CheckID":"kube_dns:4ea3f98a50351efb","CheckName":"kube_dns","CheckVersion":"2.3.0","Events":0,"ExecutionTimes":[26,17,14,15,13,14,17,14,13,17,14,12,14,13,14,17,14,13,18,13,14,12,14,13,18,14,13,13,14,13,19,12],"LastError":"","LastExecutionTime":14,"LastWarnings":[],"MetricSamples":84,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000020,"TotalRuns":33763,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439794}},"kubelet":{"kubelet:d884b5186b651429":{"AverageExecutionTime":270,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/kubelet.d/conf.yaml.default","CheckID":"kubelet:d884b5186b651429","CheckName":"kubelet","CheckVersion":"3.5.2","Events":0,"ExecutionTimes":[430,193,290,195,318,220,267,280,291,194,300,191,317,245,289,268,304,245,274,222,306,238,302,367,304,284,298,191,287,230,314,217],"LastError":"","LastExecutionTime":193,"LastWarnings":[],"MetricSamples":397,"ServiceChecks":4,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000290,"TotalRuns":33762,"TotalServiceChecks":135048,"TotalWarnings":0,"UpdateTimestamp":1584439783}},"load":{"load":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/load.d/conf.yaml.default","CheckID":"load","CheckName":"load","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202572,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439790}},"memory":{"memory":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/memory.d/conf.yaml.default","CheckID":"memory","CheckName":"memory","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":17,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":573954,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439782}},"network":{"network:e0204ad63d43c949":{"AverageExecutionTime":2,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/network.d/conf.yaml.default","CheckID":"network:e0204ad63d43c949","CheckName":"network","CheckVersion":"1.14.0","Events":0,"ExecutionTimes":[2,2,2,3,2,2,2,2,2,2,2,2,2,3,2,2,2,2,3,2,2,2,3,5,2,2,2,3,2,2,2,2],"LastError":"","LastExecutionTime":2,"LastWarnings":[],"MetricSamples":31,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000029,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439789}},"nginx":{"nginx:My Nginx Endpoints:ebb9d244d587b6a5":{"AverageExecutionTime":4,"CheckConfigSource":"kubelet:docker://91e5cce8404d9dd7ffd0acff61299c638ed7c3b908342f32dce6e7b6b2c9e59a","CheckID":"nginx:My Nginx Endpoints:ebb9d244d587b6a5","CheckName":"nginx","CheckVersion":"3.6.0","Events":0,"ExecutionTimes":[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,5,4,4,4,4,4,4,4,5,4],"LastError":"","LastExecutionTime":4,"LastWarnings":[],"MetricSamples":7,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":236341,"TotalRuns":33763,"TotalServiceChecks":33763,"TotalWarnings":0,"UpdateTimestamp":1584439795}},"ntp":{"ntp:d884b5186b651429":{"AverageExecutionTime":329,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/ntp.d/conf.yaml.default","CheckID":"ntp:d884b5186b651429","CheckName":"ntp","CheckVersion":"","Events":0,"ExecutionTimes":[170,176,274,285,215,181,574,88,88,162,170,176,239,127,114,98,159,207,263,81,391,198,5192,75,96,73,81,96,123,191,92,93],"LastError":"","LastExecutionTime":263,"LastWarnings":[],"MetricSamples":1,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":563,"TotalRuns":563,"TotalServiceChecks":563,"TotalWarnings":0,"UpdateTimestamp":1584439164}},"prometheus":{"prometheus:ksm-pods-telemetry:1048469604e4650":{"AverageExecutionTime":13,"CheckConfigSource":"kubelet:docker://13d10e998bfc81fe929a3fc61168de5e99fbe2c374c949c60a31fee38dc9896a","CheckID":"prometheus:ksm-pods-telemetry:1048469604e4650","CheckName":"prometheus","CheckVersion":"3.2.0","Events":0,"ExecutionTimes":[14,13,13,14,14,14,13,13,13,14,14,13,21,13,14,13,13,14,16,13,14,14,15,13,13,12,15,14,13,13,16,14],"LastError":"","LastExecutionTime":13,"LastWarnings":[],"MetricSamples":87,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000065,"TotalRuns":33762,"TotalServiceChecks":33762,"TotalWarnings":0,"UpdateTimestamp":1584439788}},"uptime":{"uptime":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/uptime.d/conf.yaml.default","CheckID":"uptime","CheckName":"uptime","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":1,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":33762,"TotalRuns":33762,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584439781}}},"Errors":33762,"RunningChecks":0,"Runs":473233,"Warnings":33762,"Workers":4},"time":"2020-03-17 10:09:55.392490 UTC","version":"7.17.1"}`

	twoErrorsFound = `{"JMXStatus":{"checks":{"initialized_checks":null,"failed_checks":null},"timestamp":0},"agent_metadata":{"cloud_provider":"GCP","hostname_source":"configuration"},"agent_start":"2020-03-11 13:29:20.692614 UTC","aggregatorStats":{"ChecksHistogramBucketMetricSample":0,"ChecksMetricSample":39542842,"DogstatsdMetricSample":1,"Event":13,"EventsFlushErrors":0,"EventsFlushed":13,"Flush":{"ChecksMetricSampleFlushTime":{"FlushIndex":1,"Flushes":[9930589,9694236,9613536,17816201,9385362,15089095,9551314,17001744,9856202,21628745,9570838,37656963,9816013,17850471,9941624,17810752,9962961,17034947,9856071,9672530,18031267,9644262,13711394,9779504,11596324,18534024,12072999,9819327,11611932,10348845,9696449,9624399],"LastFlush":9694236,"Name":"ChecksMetricSampleFlushTime"},"EventFlushTime":{"FlushIndex":6,"Flushes":[1151452,1116068,2710442,1582034,2199778,2347575,1182751,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":1182751,"Name":"EventFlushTime"},"MainFlushTime":{"FlushIndex":1,"Flushes":[84257,54925,76335,86944,81393,69910,81692,71407,83800,62549,87321,83394,435771,91302,73683,89260,59859,82569,74392,72080,77055,80780,80995,98125,76796,82297,70786,72576,78856,87434,71965,81850],"LastFlush":54925,"Name":"MainFlushTime"},"MetricSketchFlushTime":{"FlushIndex":-1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"MetricSketchFlushTime"},"ServiceCheckFlushTime":{"FlushIndex":1,"Flushes":[1307590,1288766,1177807,2190366,1136960,4588259,1148781,2170781,1172305,2201892,1175581,2661922,1534253,2213430,1119235,2149545,1080382,2119175,1126370,1183480,1262571,1155618,1195984,1139067,1221876,10201318,1360239,1122121,1144605,1673014,1099063,1163716],"LastFlush":1288766,"Name":"ServiceCheckFlushTime"}},"FlushCount":{"Events":{"FlushIndex":6,"Flushes":[3,1,2,2,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":2,"Name":"Events"},"Series":{"FlushIndex":1,"Flushes":[1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039],"LastFlush":1039,"Name":"Series"},"ServiceChecks":{"FlushIndex":1,"Flushes":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,22,20,20,20,20,20,20],"LastFlush":20,"Name":"ServiceChecks"},"Sketches":{"FlushIndex":1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"Sketches"}},"HostnameUpdate":0,"NumberOfFlush":33794,"SeriesFlushErrors":0,"SeriesFlushed":33592914,"ServiceCheck":643215,"ServiceCheckFlushErrors":0,"ServiceCheckFlushed":676997,"SketchesFlushErrors":0,"SketchesFlushed":0},"autoConfigStats":{"ConfigErrors":{},"ResolveWarnings":{"apache":["No service found with this AD identifier: httpd","Can't resolve the template for apache at this moment."],"cilium":["No service found with this AD identifier: cilium-agent","Can't resolve the template for cilium at this moment."],"consul":["No service found with this AD identifier: consul","Can't resolve the template for consul at this moment."],"coredns":["No service found with this AD identifier: coredns","Can't resolve the template for coredns at this moment."],"couch":["No service found with this AD identifier: couchdb","Can't resolve the template for couch at this moment."],"couchbase":["No service found with this AD identifier: couchbase","Can't resolve the template for couchbase at this moment."],"elastic":["No service found with this AD identifier: elasticsearch","No service found with this AD identifier: elasticsearch-oss","Can't resolve the template for elastic at this moment."],"etcd":["No service found with this AD identifier: etcd","Can't resolve the template for etcd at this moment."],"external_dns":["No service found with this AD identifier: external-dns","Can't resolve the template for external_dns at this moment."],"harbor":["No service found with this AD identifier: nginx-photon","Can't resolve the template for harbor at this moment."],"kube_apiserver_metrics":["No service found with this AD identifier: kube-apiserver","Can't resolve the template for kube_apiserver_metrics at this moment."],"kube_dns":["No service found with this AD identifier: kubedns-amd64","No service found with this AD identifier: k8s-dns-kube-dns-amd64","Can't resolve the template for kube_dns at this moment."],"kubernetes_state":["No service found with this AD identifier: kube-state-metrics","Can't resolve the template for kubernetes_state at this moment."],"kyototycoon":["No service found with this AD identifier: kyototycoon","Can't resolve the template for kyototycoon at this moment."],"mcache":["No service found with this AD identifier: memcached","Can't resolve the template for mcache at this moment."],"presto":["No service found with this AD identifier: presto","Can't resolve the template for presto at this moment."],"riak":["No service found with this AD identifier: riak","Can't resolve the template for riak at this moment."],"tomcat":["No service found with this AD identifier: tomcat","Can't resolve the template for tomcat at this moment."]}},"build_arch":"amd64","checkSchedulerStats":{"LoaderErrors":{},"RunErrors":{}},"clusterAgentStatus":{"Endpoint":"https://10.3.245.213:5005","Version":"1.5.2+commit.60ee741"},"conf_file":"/etc/datadog-agent/datadog.yaml","config":{"additional_checksd":"/etc/datadog-agent/checks.d","confd_path":"/etc/datadog-agent/conf.d","log_file":"","log_level":"INFO"},"dogstatsdStats":{"EventPackets":0,"EventParseErrors":0,"MetricPackets":0,"MetricParseErrors":0,"ServiceCheckPackets":0,"ServiceCheckParseErrors":0,"UdpBytes":0,"UdpPacketReadingErrors":0,"UdpPackets":1,"UdsBytes":0,"UdsOriginDetectionErrors":0,"UdsPacketReadingErrors":0,"UdsPackets":0},"endpointsInfos":{"https://app.datadoghq.com":["XXXXX"]},"forwarderStats":{"APIKeyStatus":{"API key ending with XXXXX":"Unable to validate API Key"},"Transactions":{"CheckRunsV1":33794,"Dropped":71225,"DroppedOnInput":0,"Errors":17097,"ErrorsByType":{"ConnectionErrors":0,"DNSErrors":17097,"SentRequestErrors":0,"TLSErrors":0,"WroteRequestErrors":0},"Events":0,"HTTPErrors":0,"HTTPErrorsByCode":{},"HostMetadata":0,"IntakeV1":3667,"Metadata":0,"Requeued":2357902,"Retried":182761,"RetryQueueSize":30,"Series":0,"ServiceChecks":0,"SketchSeries":0,"Success":0,"TimeseriesV1":33794}},"go_version":"go1.12.9","hostTags":["cluster_name:cluster-XXXXX","XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-qfnt.c.XXXXX-XXXXX.internal","instance-id:XXXXX","project:XXXXX-XXXXX","numeric_project_id:958371799887","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-location:europe-west1-b","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-name:XXXXX","enable-oslogin:false","google-compute-enable-pcid:true"],"hostinfo":{"hostname":"datadog-agent-x7kdh","uptime":53105,"bootTime":1583880262,"procs":57,"os":"linux","platform":"debian","platformFamily":"debian","platformVersion":"bullseye/sid","kernelVersion":"4.14.138+","virtualizationSystem":"","virtualizationRole":"","hostid":"12d3d89f-129b-7e80-b942-778e0db6610e"},"hostnameStats":{"errors":{},"provider":"configuration"},"inventories":{},"logsStats":{"is_running":false,"endpoints":null,"metrics":null,"integrations":null,"errors":null,"warnings":null},"metadata":{"os":"linux","python":"3.7.6 (default, Feb 20 2020, 13:26:46) [GCC 4.7.2]","systemStats":{"cpuCores":1,"machine":"amd64","platform":"linux","pythonV":"3.7.6","processor":"Intel(R) Xeon(R) CPU @ 2.30GHz","macV":["","",""],"nixV":["debian","bullseye/sid",""],"fbsdV":["","",""],"winV":["","",""]},"meta":{"socket-hostname":"datadog-agent-x7kdh","timezones":["UTC"],"socket-fqdn":"datadog-agent-x7kdh","ec2-hostname":"","hostname":"XXXXX-4658d5d4-qfnt","host_aliases":["XXXXX-4658d5d4-qfnt.XXXXX-XXXXX","XXXXX-4658d5d4-qfnt-cluster-XXXXX"],"instance-id":""},"host-tags":{"system":["cluster_name:cluster-XXXXX"],"google cloud platform":["XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-qfnt.c.XXXXX-XXXXX.internal","instance-id:XXXXX","project:XXXXX-XXXXX","numeric_project_id:958371799887","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-location:europe-west1-b","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-name:XXXXX","enable-oslogin:false","google-compute-enable-pcid:true"]},"container-meta":{"docker_swarm":"inactive","docker_version":"18.09.7","kubelet_version":"v1.13.12-XXXXX.25"},"network":{"network-id":"projects/958371799887/networks/default"}},"ntpOffset":0,"pid":1,"pyLoaderStats":{"ConfigureErrors":{},"Py3Warnings":{}},"pythonInit":{"Errors":[]},"python_version":"3.7.6","runnerStats":{"Checks":{"cpu":{"cpu":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cpu.d/conf.yaml.default","CheckID":"cpu","CheckName":"cpu","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202764,"TotalRuns":33795,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440278}},"cri":{"cri":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cri.d/conf.yaml.default","CheckID":"cri","CheckName":"cri","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"permanent failure in criutil: retry number exceeded","LastExecutionTime":0,"LastWarnings":["Error initialising check: permanent failure in criutil: retry number exceeded"],"MetricSamples":0,"ServiceChecks":0,"TotalErrors":33794,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":33794,"UpdateTimestamp":1584440270}},"disk":{"disk:e5dffb8bef24336f":{"AverageExecutionTime":22,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/disk.d/conf.yaml.default","CheckID":"disk:e5dffb8bef24336f","CheckName":"disk","CheckVersion":"2.6.0","Events":0,"ExecutionTimes":[20,23,23,21,30,21,23,22,30,21,21,21,22,21,27,21,21,21,21,21,21,23,21,21,21,23,24,21,22,21,21,23],"LastError":"","LastExecutionTime":23,"LastWarnings":[],"MetricSamples":228,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000008,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440277}},"docker":{"docker":{"AverageExecutionTime":38,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/docker.d/conf.yaml.default","CheckID":"docker","CheckName":"docker","CheckVersion":"","Events":0,"ExecutionTimes":[40,37,40,31,38,32,63,31,35,31,37,33,39,33,38,32,38,32,57,35,33,60,36,57,31,31,40,33,32,33,42,37],"LastError":"","LastExecutionTime":37,"LastWarnings":[],"MetricSamples":250,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":12,"TotalMetricSamples":1000050,"TotalRuns":33794,"TotalServiceChecks":33794,"TotalWarnings":0,"UpdateTimestamp":1584440269}},"file_handle":{"file_handle":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/file_handle.d/conf.yaml.default","CheckID":"file_handle","CheckName":"file_handle","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":5,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":168970,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440276}},"io":{"io":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/io.d/conf.yaml.default","CheckID":"io","CheckName":"io","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":182,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000146,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440268}},"kubelet":{"kubelet:d884b5186b651429":{"AverageExecutionTime":253,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/kubelet.d/conf.yaml.default","CheckID":"kubelet:d884b5186b651429","CheckName":"kubelet","CheckVersion":"3.5.2","Events":0,"ExecutionTimes":[202,258,248,255,226,248,243,285,201,258,300,255,220,292,259,254,242,319,283,301,217,283,264,256,207,305,199,253,223,238,182,327],"LastError":"","LastExecutionTime":258,"LastWarnings":[],"MetricSamples":297,"ServiceChecks":4,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000097,"TotalRuns":33794,"TotalServiceChecks":135176,"TotalWarnings":0,"UpdateTimestamp":1584440275}},"load":{"load":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/load.d/conf.yaml.default","CheckID":"load","CheckName":"load","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202764,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440267}},"memory":{"memory":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/memory.d/conf.yaml.default","CheckID":"memory","CheckName":"memory","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":17,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":574498,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440274}},"network":{"network:e0204ad63d43c949":{"AverageExecutionTime":2,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/network.d/conf.yaml.default","CheckID":"network:e0204ad63d43c949","CheckName":"network","CheckVersion":"1.14.0","Events":0,"ExecutionTimes":[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2],"LastError":"","LastExecutionTime":2,"LastWarnings":[],"MetricSamples":31,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000029,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440266}},"ntp":{"ntp:d884b5186b651429":{"AverageExecutionTime":80066,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/ntp.d/conf.yaml.default","CheckID":"ntp:d884b5186b651429","CheckName":"ntp","CheckVersion":"","Events":0,"ExecutionTimes":[80067,80068,80056,80067,80061,80070,80069,80068,80064,80063,80062,80071,80076,80062,80070,80070,80058,80066,80071,80057,80071,80073,80066,80063,80066,80074,80062,80070,80059,80066,80070,80066],"LastError":"","LastExecutionTime":80057,"LastWarnings":[],"MetricSamples":0,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":564,"TotalServiceChecks":564,"TotalWarnings":0,"UpdateTimestamp":1584440148}},"openmetrics":{"openmetrics:datadog.operator:94da93062e6598f9":{"AverageExecutionTime":18,"CheckConfigSource":"kubelet:docker://5ff0c1159d321881ed0d347fc0bed52eb47d3772fd32381e0d7bec0c096ae518","CheckID":"openmetrics:datadog.operator:94da93062e6598f9","CheckName":"openmetrics","CheckVersion":"1.4.0","Events":0,"ExecutionTimes":[18,18,18,18,18,18,18,18,18,19,18,18,19,30,18,21,18,19,26,17,18,19,17,18,19,18,18,18,18,18,18,18],"LastError":"","LastExecutionTime":18,"LastWarnings":[],"MetricSamples":167,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000082,"TotalRuns":33794,"TotalServiceChecks":33794,"TotalWarnings":0,"UpdateTimestamp":1584440272}},"redisdb":{"redisdb:766ed21d64724d11":{"AverageExecutionTime":0,"CheckConfigSource":"kubelet:docker://338d4db7de231c8a9f72a62bd755c6d6a4597b8e9e88260244361fe90598165b","CheckID":"redisdb:766ed21d64724d11","CheckName":"redisdb","CheckVersion":"1.15.0","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"[{\"message\": \"You must specify a host/port couple or a unix_socket_path\", \"traceback\": \"Traceback (most recent call last):\\n  File \\\"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/base/checks/base.py\\\", line 673, in run\\n    self.check(instance)\\n  File \\\"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/redisdb/redisdb.py\\\", line 481, in check\\n    raise Exception(\\\"You must specify a host/port couple or a unix_socket_path\\\")\\nException: You must specify a host/port couple or a unix_socket_path\\n\"}]","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":0,"ServiceChecks":0,"TotalErrors":33794,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440265}},"uptime":{"uptime":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/uptime.d/conf.yaml.default","CheckID":"uptime","CheckName":"uptime","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":1,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":33794,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440273}}},"Errors":67588,"RunningChecks":0,"Runs":439887,"Warnings":33794,"Workers":4},"time":"2020-03-17 10:18:00.061901 UTC","version":"7.17.1"}`

	invalidPayload = `{"JMXStatus":{"checks":{"initialized_checks":null,"failed_checks":null},"timestamp":0},"agent_metadata":{"cloud_provider":"GCP","hostname_source":"configuration"},"agent_start":"2020-03-11 13:29:20.692614 UTC","aggregatorStats":{"ChecksHistogramBucketMetricSample":0,"ChecksMetricSample":39542842,"DogstatsdMetricSample":1,"Event":13,"EventsFlushErrors":0,"EventsFlushed":13,"Flush":{"ChecksMetricSampleFlushTime":{"FlushIndex":1,"Flushes":[9930589,9694236,9613536,17816201,9385362,15089095,9551314,17001744,9856202,21628745,9570838,37656963,9816013,17850471,9941624,17810752,9962961,17034947,9856071,9672530,18031267,9644262,13711394,9779504,11596324,18534024,12072999,9819327,11611932,10348845,9696449,9624399],"LastFlush":9694236,"Name":"ChecksMetricSampleFlushTime"},"EventFlushTime":{"FlushIndex":6,"Flushes":[1151452,1116068,2710442,1582034,2199778,2347575,1182751,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":1182751,"Name":"EventFlushTime"},"MainFlushTime":{"FlushIndex":1,"Flushes":[84257,54925,76335,86944,81393,69910,81692,71407,83800,62549,87321,83394,435771,91302,73683,89260,59859,82569,74392,72080,77055,80780,80995,98125,76796,82297,70786,72576,78856,87434,71965,81850],"LastFlush":54925,"Name":"MainFlushTime"},"MetricSketchFlushTime":{"FlushIndex":-1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"MetricSketchFlushTime"},"ServiceCheckFlushTime":{"FlushIndex":1,"Flushes":[1307590,1288766,1177807,2190366,1136960,4588259,1148781,2170781,1172305,2201892,1175581,2661922,1534253,2213430,1119235,2149545,1080382,2119175,1126370,1183480,1262571,1155618,1195984,1139067,1221876,10201318,1360239,1122121,1144605,1673014,1099063,1163716],"LastFlush":1288766,"Name":"ServiceCheckFlushTime"}},"FlushCount":{"Events":{"FlushIndex":6,"Flushes":[3,1,2,2,1,2,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":2,"Name":"Events"},"Series":{"FlushIndex":1,"Flushes":[1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039,1039],"LastFlush":1039,"Name":"Series"},"ServiceChecks":{"FlushIndex":1,"Flushes":[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,22,20,20,20,20,20,20],"LastFlush":20,"Name":"ServiceChecks"},"Sketches":{"FlushIndex":1,"Flushes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastFlush":0,"Name":"Sketches"}},"HostnameUpdate":0,"NumberOfFlush":33794,"SeriesFlushErrors":0,"SeriesFlushed":33592914,"ServiceCheck":643215,"ServiceCheckFlushErrors":0,"ServiceCheckFlushed":676997,"SketchesFlushErrors":0,"SketchesFlushed":0},"autoConfigStats":{"ConfigErrors":{},"ResolveWarnings":{"apache":["No service found with this AD identifier: httpd","Can't resolve the template for apache at this moment."],"cilium":["No service found with this AD identifier: cilium-agent","Can't resolve the template for cilium at this moment."],"consul":["No service found with this AD identifier: consul","Can't resolve the template for consul at this moment."],"coredns":["No service found with this AD identifier: coredns","Can't resolve the template for coredns at this moment."],"couch":["No service found with this AD identifier: couchdb","Can't resolve the template for couch at this moment."],"couchbase":["No service found with this AD identifier: couchbase","Can't resolve the template for couchbase at this moment."],"elastic":["No service found with this AD identifier: elasticsearch","No service found with this AD identifier: elasticsearch-oss","Can't resolve the template for elastic at this moment."],"etcd":["No service found with this AD identifier: etcd","Can't resolve the template for etcd at this moment."],"external_dns":["No service found with this AD identifier: external-dns","Can't resolve the template for external_dns at this moment."],"harbor":["No service found with this AD identifier: nginx-photon","Can't resolve the template for harbor at this moment."],"kube_apiserver_metrics":["No service found with this AD identifier: kube-apiserver","Can't resolve the template for kube_apiserver_metrics at this moment."],"kube_dns":["No service found with this AD identifier: kubedns-amd64","No service found with this AD identifier: k8s-dns-kube-dns-amd64","Can't resolve the template for kube_dns at this moment."],"kubernetes_state":["No service found with this AD identifier: kube-state-metrics","Can't resolve the template for kubernetes_state at this moment."],"kyototycoon":["No service found with this AD identifier: kyototycoon","Can't resolve the template for kyototycoon at this moment."],"mcache":["No service found with this AD identifier: memcached","Can't resolve the template for mcache at this moment."],"presto":["No service found with this AD identifier: presto","Can't resolve the template for presto at this moment."],"riak":["No service found with this AD identifier: riak","Can't resolve the template for riak at this moment."],"tomcat":["No service found with this AD identifier: tomcat","Can't resolve the template for tomcat at this moment."]}},"build_arch":"amd64","checkSchedulerStats":{"LoaderErrors":{},"RunErrors":{}},"clusterAgentStatus":{"Endpoint":"https://10.3.245.213:5005","Version":"1.5.2+commit.60ee741"},"conf_file":"/etc/datadog-agent/datadog.yaml","config":{"additional_checksd":"/etc/datadog-agent/checks.d","confd_path":"/etc/datadog-agent/conf.d","log_file":"","log_level":"INFO"},"dogstatsdStats":{"EventPackets":0,"EventParseErrors":0,"MetricPackets":0,"MetricParseErrors":0,"ServiceCheckPackets":0,"ServiceCheckParseErrors":0,"UdpBytes":0,"UdpPacketReadingErrors":0,"UdpPackets":1,"UdsBytes":0,"UdsOriginDetectionErrors":0,"UdsPacketReadingErrors":0,"UdsPackets":0},"endpointsInfos":{"https://app.datadoghq.com":["XXXXX"]},"forwarderStats":{"APIKeyStatus":{"API key ending with XXXXX":"Unable to validate API Key"},"Transactions":{"CheckRunsV1":33794,"Dropped":71225,"DroppedOnInput":0,"Errors":17097,"ErrorsByType":{"ConnectionErrors":0,"DNSErrors":17097,"SentRequestErrors":0,"TLSErrors":0,"WroteRequestErrors":0},"Events":0,"HTTPErrors":0,"HTTPErrorsByCode":{},"HostMetadata":0,"IntakeV1":3667,"Metadata":0,"Requeued":2357902,"Retried":182761,"RetryQueueSize":30,"Series":0,"ServiceChecks":0,"SketchSeries":0,"Success":0,"TimeseriesV1":33794}},"go_version":"go1.12.9","hostTags":["cluster_name:cluster-XXXXX","XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-qfnt.c.XXXXX-XXXXX.internal","instance-id:XXXXX","project:XXXXX-XXXXX","numeric_project_id:958371799887","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-location:europe-west1-b","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-name:XXXXX","enable-oslogin:false","google-compute-enable-pcid:true"],"hostinfo":{"hostname":"datadog-agent-x7kdh","uptime":53105,"bootTime":1583880262,"procs":57,"os":"linux","platform":"debian","platformFamily":"debian","platformVersion":"bullseye/sid","kernelVersion":"4.14.138+","virtualizationSystem":"","virtualizationRole":"","hostid":"12d3d89f-129b-7e80-b942-778e0db6610e"},"hostnameStats":{"errors":{},"provider":"configuration"},"inventories":{},"logsStats":{"is_running":false,"endpoints":null,"metrics":null,"integrations":null,"errors":null,"warnings":null},"metadata":{"os":"linux","python":"3.7.6 (default, Feb 20 2020, 13:26:46) [GCC 4.7.2]","systemStats":{"cpuCores":1,"machine":"amd64","platform":"linux","pythonV":"3.7.6","processor":"Intel(R) Xeon(R) CPU @ 2.30GHz","macV":["","",""],"nixV":["debian","bullseye/sid",""],"fbsdV":["","",""],"winV":["","",""]},"meta":{"socket-hostname":"datadog-agent-x7kdh","timezones":["UTC"],"socket-fqdn":"datadog-agent-x7kdh","ec2-hostname":"","hostname":"XXXXX-4658d5d4-qfnt","host_aliases":["XXXXX-4658d5d4-qfnt.XXXXX-XXXXX","XXXXX-4658d5d4-qfnt-cluster-XXXXX"],"instance-id":""},"host-tags":{"system":["cluster_name:cluster-XXXXX"],"google cloud platform":["XXXXX-XXXXX-8a009fe2-node","zone:europe-west1-b","instance-type:n1-standard-1","internal-hostname:XXXXX-4658d5d4-qfnt.c.XXXXX-XXXXX.internal","instance-id:XXXXX","project:XXXXX-XXXXX","numeric_project_id:958371799887","cluster-uid:XXXXX","gci-ensure-XXXXX-docker:true","instance-template:projects/958371799887/global/instanceTemplates/XXXXX-e8bb55d2","cluster-location:europe-west1-b","created-by:projects/958371799887/zones/europe-west1-b/instanceGroupManagers/XXXXX-4658d5d4-grp","gci-update-strategy:update_disabled","kube-labels:beta.kubernetes.io/fluentd-ds-ready=true,cloud.google.com/XXXXX-nodepool=default-pool,cloud.google.com/XXXXX-os-distribution=cos","cluster-name:XXXXX","enable-oslogin:false","google-compute-enable-pcid:true"]},"container-meta":{"docker_swarm":"inactive","docker_version":"18.09.7","kubelet_version":"v1.13.12-XXXXX.25"},"network":{"network-id":"projects/958371799887/networks/default"}},"ntpOffset":0,"pid":1,"pyLoaderStats":{"ConfigureErrors":{},"Py3Warnings":{}},"pythonInit":{"Errors":[]},"python_version":"3.7.6","runnerStats":"Checks":{"cpu":{"cpu":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cpu.d/conf.yaml.default","CheckID":"cpu","CheckName":"cpu","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202764,"TotalRuns":33795,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440278}},"cri":{"cri":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/cri.d/conf.yaml.default","CheckID":"cri","CheckName":"cri","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"permanent failure in criutil: retry number exceeded","LastExecutionTime":0,"LastWarnings":["Error initialising check: permanent failure in criutil: retry number exceeded"],"MetricSamples":0,"ServiceChecks":0,"TotalErrors":33794,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":33794,"UpdateTimestamp":1584440270}},"disk":{"disk:e5dffb8bef24336f":{"AverageExecutionTime":22,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/disk.d/conf.yaml.default","CheckID":"disk:e5dffb8bef24336f","CheckName":"disk","CheckVersion":"2.6.0","Events":0,"ExecutionTimes":[20,23,23,21,30,21,23,22,30,21,21,21,22,21,27,21,21,21,21,21,21,23,21,21,21,23,24,21,22,21,21,23],"LastError":"","LastExecutionTime":23,"LastWarnings":[],"MetricSamples":228,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000008,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440277}},"docker":{"docker":{"AverageExecutionTime":38,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/docker.d/conf.yaml.default","CheckID":"docker","CheckName":"docker","CheckVersion":"","Events":0,"ExecutionTimes":[40,37,40,31,38,32,63,31,35,31,37,33,39,33,38,32,38,32,57,35,33,60,36,57,31,31,40,33,32,33,42,37],"LastError":"","LastExecutionTime":37,"LastWarnings":[],"MetricSamples":250,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":12,"TotalMetricSamples":1000050,"TotalRuns":33794,"TotalServiceChecks":33794,"TotalWarnings":0,"UpdateTimestamp":1584440269}},"file_handle":{"file_handle":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/file_handle.d/conf.yaml.default","CheckID":"file_handle","CheckName":"file_handle","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":5,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":168970,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440276}},"io":{"io":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/io.d/conf.yaml.default","CheckID":"io","CheckName":"io","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":182,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000146,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440268}},"kubelet":{"kubelet:d884b5186b651429":{"AverageExecutionTime":253,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/kubelet.d/conf.yaml.default","CheckID":"kubelet:d884b5186b651429","CheckName":"kubelet","CheckVersion":"3.5.2","Events":0,"ExecutionTimes":[202,258,248,255,226,248,243,285,201,258,300,255,220,292,259,254,242,319,283,301,217,283,264,256,207,305,199,253,223,238,182,327],"LastError":"","LastExecutionTime":258,"LastWarnings":[],"MetricSamples":297,"ServiceChecks":4,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000097,"TotalRuns":33794,"TotalServiceChecks":135176,"TotalWarnings":0,"UpdateTimestamp":1584440275}},"load":{"load":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/load.d/conf.yaml.default","CheckID":"load","CheckName":"load","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":6,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":202764,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440267}},"memory":{"memory":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/memory.d/conf.yaml.default","CheckID":"memory","CheckName":"memory","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":17,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":574498,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440274}},"network":{"network:e0204ad63d43c949":{"AverageExecutionTime":2,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/network.d/conf.yaml.default","CheckID":"network:e0204ad63d43c949","CheckName":"network","CheckVersion":"1.14.0","Events":0,"ExecutionTimes":[2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2],"LastError":"","LastExecutionTime":2,"LastWarnings":[],"MetricSamples":31,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000029,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440266}},"ntp":{"ntp:d884b5186b651429":{"AverageExecutionTime":80066,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/ntp.d/conf.yaml.default","CheckID":"ntp:d884b5186b651429","CheckName":"ntp","CheckVersion":"","Events":0,"ExecutionTimes":[80067,80068,80056,80067,80061,80070,80069,80068,80064,80063,80062,80071,80076,80062,80070,80070,80058,80066,80071,80057,80071,80073,80066,80063,80066,80074,80062,80070,80059,80066,80070,80066],"LastError":"","LastExecutionTime":80057,"LastWarnings":[],"MetricSamples":0,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":564,"TotalServiceChecks":564,"TotalWarnings":0,"UpdateTimestamp":1584440148}},"openmetrics":{"openmetrics:datadog.operator:94da93062e6598f9":{"AverageExecutionTime":18,"CheckConfigSource":"kubelet:docker://5ff0c1159d321881ed0d347fc0bed52eb47d3772fd32381e0d7bec0c096ae518","CheckID":"openmetrics:datadog.operator:94da93062e6598f9","CheckName":"openmetrics","CheckVersion":"1.4.0","Events":0,"ExecutionTimes":[18,18,18,18,18,18,18,18,18,19,18,18,19,30,18,21,18,19,26,17,18,19,17,18,19,18,18,18,18,18,18,18],"LastError":"","LastExecutionTime":18,"LastWarnings":[],"MetricSamples":167,"ServiceChecks":1,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":1000082,"TotalRuns":33794,"TotalServiceChecks":33794,"TotalWarnings":0,"UpdateTimestamp":1584440272}},"redisdb":{"redisdb:766ed21d64724d11":{"AverageExecutionTime":0,"CheckConfigSource":"kubelet:docker://338d4db7de231c8a9f72a62bd755c6d6a4597b8e9e88260244361fe90598165b","CheckID":"redisdb:766ed21d64724d11","CheckName":"redisdb","CheckVersion":"1.15.0","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"[{\"message\": \"You must specify a host/port couple or a unix_socket_path\", \"traceback\": \"Traceback (most recent call last):\\n  File \\\"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/base/checks/base.py\\\", line 673, in run\\n    self.check(instance)\\n  File \\\"/opt/datadog-agent/embedded/lib/python3.7/site-packages/datadog_checks/redisdb/redisdb.py\\\", line 481, in check\\n    raise Exception(\\\"You must specify a host/port couple or a unix_socket_path\\\")\\nException: You must specify a host/port couple or a unix_socket_path\\n\"}]","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":0,"ServiceChecks":0,"TotalErrors":33794,"TotalEvents":0,"TotalMetricSamples":0,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440265}},"uptime":{"uptime":{"AverageExecutionTime":0,"CheckConfigSource":"file:/etc/datadog-agent/conf.d/uptime.d/conf.yaml.default","CheckID":"uptime","CheckName":"uptime","CheckVersion":"","Events":0,"ExecutionTimes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"LastError":"","LastExecutionTime":0,"LastWarnings":[],"MetricSamples":1,"ServiceChecks":0,"TotalErrors":0,"TotalEvents":0,"TotalMetricSamples":33794,"TotalRuns":33794,"TotalServiceChecks":0,"TotalWarnings":0,"UpdateTimestamp":1584440273}}},"Errors":67588,"RunningChecks":0,"Runs":439887,"Warnings":33794,"Workers":4},"time":"2020-03-17 10:18:00.061901 UTC","version":"7.17.1"}`
)
