From 6130b3da18530c0b735d9dd631df1ddb66319430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 13 Feb 2025 17:23:44 +0100 Subject: [PATCH] Upload Datadog average CPU usage metric in citool --- .github/workflows/ci.yml | 8 +- src/ci/citool/Cargo.lock | 680 ++++++++++++++++++++++++++++++++- src/ci/citool/Cargo.toml | 2 + src/ci/citool/src/cpu_usage.rs | 22 ++ src/ci/citool/src/datadog.rs | 43 +++ src/ci/citool/src/main.rs | 41 +- src/ci/citool/src/utils.rs | 12 + 7 files changed, 787 insertions(+), 21 deletions(-) create mode 100644 src/ci/citool/src/cpu_usage.rs create mode 100644 src/ci/citool/src/datadog.rs create mode 100644 src/ci/citool/src/utils.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be4919ce1a3..f166e0c0b41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -183,7 +183,7 @@ jobs: run: src/ci/scripts/dump-environment.sh # Pre-build citool before the following step uninstalls rustup - # Build is into the build directory, to avoid modifying sources + # Build it into the build directory, to avoid modifying sources - name: build citool run: | cd src/ci/citool @@ -238,13 +238,9 @@ jobs: - name: upload job metrics to DataDog if: needs.calculate_matrix.outputs.run_type != 'pr' env: - DATADOG_SITE: datadoghq.com DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }} DD_GITHUB_JOB_NAME: ${{ matrix.full_name }} - run: | - cd src/ci - npm ci - python3 scripts/upload-build-metrics.py ../../build/cpu-usage.csv + run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv # This job isused to tell bors the final status of the build, as there is no practical way to detect # when a workflow is successful listening to webhooks only in our current bors implementation (homu). diff --git a/src/ci/citool/Cargo.lock b/src/ci/citool/Cargo.lock index e5be2d42472..46343a7b86e 100644 --- a/src/ci/citool/Cargo.lock +++ b/src/ci/citool/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "anstream" version = "0.6.18" @@ -38,7 +44,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -49,7 +55,7 @@ checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", "once_cell", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -58,6 +64,12 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "build_helper" version = "0.1.0" @@ -66,6 +78,27 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "bytes" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" + +[[package]] +name = "cc" +version = "1.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "citool" version = "0.1.0" @@ -73,10 +106,12 @@ dependencies = [ "anyhow", "build_helper", "clap", + "csv", "insta", "serde", "serde_json", "serde_yaml", + "ureq", ] [[package]] @@ -134,7 +169,95 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", ] [[package]] @@ -149,6 +272,42 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "flate2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "hashbrown" version = "0.15.2" @@ -161,6 +320,162 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -208,18 +523,57 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + +[[package]] +name = "log" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "miniz_oxide" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +dependencies = [ + "adler2", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "once_cell" version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pin-project" version = "1.1.9" @@ -240,6 +594,12 @@ dependencies = [ "syn", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "proc-macro2" version = "1.0.93" @@ -258,6 +618,61 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ring" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.19" @@ -309,18 +724,42 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "smallvec" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.98" @@ -332,6 +771,58 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb041120f25f8fbe8fd2dbe4671c7c2ed74d83be2e7a77529bf7e0790ae3f472" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef" + +[[package]] +name = "time-macros" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8093bc3e81c3bc5f7879de09619d06c9a5a5e45ca44dfeeb7225bae38005c5c" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "unicode-ident" version = "1.0.16" @@ -344,12 +835,110 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf-8", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -422,3 +1011,88 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/src/ci/citool/Cargo.toml b/src/ci/citool/Cargo.toml index 3b325268309..c486f2977a1 100644 --- a/src/ci/citool/Cargo.toml +++ b/src/ci/citool/Cargo.toml @@ -6,9 +6,11 @@ edition = "2021" [dependencies] anyhow = "1" clap = { version = "4.5", features = ["derive"] } +csv = "1" serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +ureq = { version = "3", features = ["json"] } build_helper = { path = "../../build_helper" } diff --git a/src/ci/citool/src/cpu_usage.rs b/src/ci/citool/src/cpu_usage.rs new file mode 100644 index 00000000000..4c9e37bf6db --- /dev/null +++ b/src/ci/citool/src/cpu_usage.rs @@ -0,0 +1,22 @@ +use std::path::Path; + +/// Loads CPU usage records from a CSV generated by the `src/ci/scripts/collect-cpu-stats.sh` +/// script. +pub fn load_cpu_usage(path: &Path) -> anyhow::Result> { + let reader = csv::ReaderBuilder::new().flexible(true).from_path(path)?; + + let mut entries = vec![]; + for row in reader.into_records() { + let row = row?; + let cols = row.into_iter().collect::>(); + + // The log might contain incomplete rows or some Python exception + if cols.len() == 2 { + if let Ok(idle) = cols[1].parse::() { + entries.push(100.0 - idle); + } + } + } + + Ok(entries) +} diff --git a/src/ci/citool/src/datadog.rs b/src/ci/citool/src/datadog.rs new file mode 100644 index 00000000000..837257420d6 --- /dev/null +++ b/src/ci/citool/src/datadog.rs @@ -0,0 +1,43 @@ +use anyhow::Context; + +use crate::utils::load_env_var; + +/// Uploads a custom CI pipeline metric to Datadog. +/// Expects to be executed from within the context of a GitHub Actions job. +pub fn upload_datadog_metric(name: &str, value: f64) -> anyhow::Result<()> { + let datadog_api_key = load_env_var("DATADOG_API_KEY")?; + let github_server_url = load_env_var("GITHUB_SERVER_URL")?; + let github_repository = load_env_var("GITHUB_REPOSITORY")?; + let github_run_id = load_env_var("GITHUB_RUN_ID")?; + let github_run_attempt = load_env_var("GITHUB_RUN_ATTEMPT")?; + let github_job = load_env_var("GITHUB_JOB")?; + let dd_github_job_name = load_env_var("DD_GITHUB_JOB_NAME")?; + + // This API endpoint is not documented in Datadog's API reference currently. + // It was reverse-engineered from the `datadog-ci measure` npm command. + ureq::post("https://api.datadoghq.com/api/v2/ci/pipeline/metrics") + .header("DD-API-KEY", datadog_api_key) + .send_json(serde_json::json!({ + "data": { + "attributes": { + "ci_env": { + "GITHUB_SERVER_URL": github_server_url, + "GITHUB_REPOSITORY": github_repository, + "GITHUB_RUN_ID": github_run_id, + "GITHUB_RUN_ATTEMPT": github_run_attempt, + "GITHUB_JOB": github_job, + "DD_GITHUB_JOB_NAME": dd_github_job_name + }, + // Job level + "ci_level": 1, + "metrics": { + name: value + }, + "provider": "github" + }, + "type": "ci_custom_metric" + } + })) + .context("cannot send metric to DataDog")?; + Ok(()) +} diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs index cef92e998da..a1e5658be36 100644 --- a/src/ci/citool/src/main.rs +++ b/src/ci/citool/src/main.rs @@ -1,4 +1,7 @@ +mod cpu_usage; +mod datadog; mod metrics; +mod utils; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -8,7 +11,10 @@ use anyhow::Context; use clap::Parser; use serde_yaml::Value; +use crate::cpu_usage::load_cpu_usage; +use crate::datadog::upload_datadog_metric; use crate::metrics::postprocess_metrics; +use crate::utils::load_env_var; const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/.."); const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker"); @@ -75,7 +81,7 @@ impl JobDatabase { } fn load_job_db(path: &Path) -> anyhow::Result { - let db = read_to_string(path)?; + let db = utils::read_to_string(path)?; let mut db: Value = serde_yaml::from_str(&db)?; // We need to expand merge keys (<<), because serde_yaml can't deal with them @@ -148,10 +154,6 @@ impl GitHubContext { } } -fn load_env_var(name: &str) -> anyhow::Result { - std::env::var(name).with_context(|| format!("Cannot find variable {name}")) -} - fn load_github_ctx() -> anyhow::Result { let event_name = load_env_var("GITHUB_EVENT_NAME")?; let commit_message = @@ -325,6 +327,18 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any if !result.success() { Err(anyhow::anyhow!("Job failed")) } else { Ok(()) } } +fn upload_ci_metrics(cpu_usage_csv: &Path) -> anyhow::Result<()> { + let usage = load_cpu_usage(cpu_usage_csv).context("Cannot load CPU usage from input CSV")?; + eprintln!("CPU usage\n{usage:?}"); + + let avg = if !usage.is_empty() { usage.iter().sum::() / usage.len() as f64 } else { 0.0 }; + eprintln!("CPU usage average: {avg}"); + + upload_datadog_metric("avg-cpu-usage", avg).context("Cannot upload Datadog metric")?; + + Ok(()) +} + #[derive(clap::Parser)] enum Args { /// Calculate a list of jobs that should be executed on CI. @@ -350,6 +364,11 @@ enum Args { /// Usually, this will be GITHUB_STEP_SUMMARY on CI. summary_path: PathBuf, }, + /// Upload CI metrics to Datadog. + UploadBuildMetrics { + /// Path to a CSV containing the CI job CPU usage. + cpu_usage_csv: PathBuf, + }, } #[derive(clap::ValueEnum, Clone)] @@ -370,7 +389,7 @@ fn main() -> anyhow::Result<()> { let jobs_path = jobs_file.as_deref().unwrap_or(default_jobs_file); let gh_ctx = load_github_ctx() .context("Cannot load environment variables from GitHub Actions")?; - let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel")) + let channel = utils::read_to_string(Path::new(CI_DIRECTORY).join("channel")) .context("Cannot read channel file")? .trim() .to_string(); @@ -379,7 +398,10 @@ fn main() -> anyhow::Result<()> { .context("Failed to calculate job matrix")?; } Args::RunJobLocally { job_type, name } => { - run_workflow_locally(load_db(default_jobs_file)?, job_type, name)? + run_workflow_locally(load_db(default_jobs_file)?, job_type, name)?; + } + Args::UploadBuildMetrics { cpu_usage_csv } => { + upload_ci_metrics(&cpu_usage_csv)?; } Args::PostprocessMetrics { metrics_path, summary_path } => { postprocess_metrics(&metrics_path, &summary_path)?; @@ -388,8 +410,3 @@ fn main() -> anyhow::Result<()> { Ok(()) } - -fn read_to_string>(path: P) -> anyhow::Result { - let error = format!("Cannot read file {:?}", path.as_ref()); - std::fs::read_to_string(path).context(error) -} diff --git a/src/ci/citool/src/utils.rs b/src/ci/citool/src/utils.rs new file mode 100644 index 00000000000..eb268ca397c --- /dev/null +++ b/src/ci/citool/src/utils.rs @@ -0,0 +1,12 @@ +use std::path::Path; + +use anyhow::Context; + +pub fn load_env_var(name: &str) -> anyhow::Result { + std::env::var(name).with_context(|| format!("Cannot find environment variable `{name}`")) +} + +pub fn read_to_string>(path: P) -> anyhow::Result { + let error = format!("Cannot read file {:?}", path.as_ref()); + std::fs::read_to_string(path).context(error) +}