From 8f1ef98427df78448ddc4c0786fd96d076c33c4a Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Fri, 2 Oct 2026 07:42:53 -0700 Subject: [PATCH 1/2] Create and upload attack data archive file to new path using contentctl-ng --- .github/workflows/mirror_data_archive.yml | 40 +++++++++++++++-------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/.github/workflows/mirror_data_archive.yml b/.github/workflows/mirror_data_archive.yml index 7cf9571de..994ee399a 100644 --- a/.github/workflows/mirror_data_archive.yml +++ b/.github/workflows/mirror_data_archive.yml @@ -11,7 +11,7 @@ jobs: group: attack-data-runners env: BUCKET: attack-range-attack-data - ATTACK_DATA_ARCHIVE_FILE: attack_data.tar.zstd + NEW_ATTACK_DATA_ARCHIVE_FILE: attack_data_archive/attack_data_archive.zip steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -19,28 +19,40 @@ jobs: with: lfs: true + - name: Setup Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Checkout security_content Repo + uses: actions/checkout@v6 + with: + repository: splunk/security_content + path: security_content + + - name: Install contentctl-ng + run: | + pip install -c security_content/requirements.txt contentctl-ng + - name: Setup AWS CLI and Credentials uses: aws-actions/configure-aws-credentials@v6 with: - aws-access-key-id: ${{ secrets.ACCESS_KEY}} + aws-access-key-id: ${{ secrets.ACCESS_KEY }} aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} aws-region: us-west-2 - name: Create archive of ONLY the datasets folder run: | - # The structure of the tar + zstd archive should mirror that of checking out the repo directly - mkdir attack_data - mv datasets/ attack_data/. - - #Build some metadata about the archive for documentation purposes - git rev-parse HEAD > attack_data/git_hash.txt - date -u > attack_data/cache_build_date.txt + # Use contentctl-ng to create the Attack Data Archive File + # There is no need to make any other changes to the repo + contentctl-ng build-attack-data-archive - # Compress with number of threads equal to number of CPU cores. - # Compression level 10 is a great compromise of speed and file size. - # File size reductions are diminishing returns after this - determined experimentally. - tar -c attack_data | zstd --compress -T0 -10 -o $ATTACK_DATA_ARCHIVE_FILE + #This command builds the files: + # - attack_data_archive/attack_data_archive.zip + # - attack_data_archive/metadata.yml + # We intentionally only upload the attack_data_archive.zip file to S3 + ls -lh attack_data_archive - name: Upload Attack data archive file to S3 Bucket run: | - aws s3 cp $ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/ + aws s3 cp $NEW_ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/ From 070aad8e8798fcac7da79560d1b80ccb0d6fd689 Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 15 Oct 2026 18:08:04 -0700 Subject: [PATCH 2/2] Keep uploading legacy attack_data.tar.zstd archive for backward compatibility Co-Authored-By: Claude Code --- .github/workflows/mirror_data_archive.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/mirror_data_archive.yml b/.github/workflows/mirror_data_archive.yml index 994ee399a..70a8f44af 100644 --- a/.github/workflows/mirror_data_archive.yml +++ b/.github/workflows/mirror_data_archive.yml @@ -12,6 +12,7 @@ jobs: env: BUCKET: attack-range-attack-data NEW_ATTACK_DATA_ARCHIVE_FILE: attack_data_archive/attack_data_archive.zip + LEGACY_ATTACK_DATA_ARCHIVE_FILE: attack_data.tar.zstd steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -56,3 +57,24 @@ jobs: - name: Upload Attack data archive file to S3 Bucket run: | aws s3 cp $NEW_ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/ + + # Backward compatibility: keep publishing the original tar+zstd archive at the original path. + # This MUST run after the contentctl-ng build because it moves the datasets/ folder. + - name: Create legacy tar+zstd archive of ONLY the datasets folder + run: | + # The structure of the tar + zstd archive should mirror that of checking out the repo directly + mkdir attack_data + mv datasets/ attack_data/. + + #Build some metadata about the archive for documentation purposes + git rev-parse HEAD > attack_data/git_hash.txt + date -u > attack_data/cache_build_date.txt + + # Compress with number of threads equal to number of CPU cores. + # Compression level 10 is a great compromise of speed and file size. + # File size reductions are diminishing returns after this - determined experimentally. + tar -c attack_data | zstd --compress -T0 -10 -o $LEGACY_ATTACK_DATA_ARCHIVE_FILE + + - name: Upload legacy Attack data archive file to S3 Bucket + run: | + aws s3 cp $LEGACY_ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/