diff --git a/.github/workflows/mirror_data_archive.yml b/.github/workflows/mirror_data_archive.yml index 7cf9571d..70a8f44a 100644 --- a/.github/workflows/mirror_data_archive.yml +++ b/.github/workflows/mirror_data_archive.yml @@ -11,7 +11,8 @@ jobs: group: attack-data-runners env: BUCKET: attack-range-attack-data - ATTACK_DATA_ARCHIVE_FILE: attack_data.tar.zstd + NEW_ATTACK_DATA_ARCHIVE_FILE: attack_data_archive/attack_data_archive.zip + LEGACY_ATTACK_DATA_ARCHIVE_FILE: attack_data.tar.zstd steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -19,14 +20,47 @@ jobs: with: lfs: true + - name: Setup Python 3.14 + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Checkout security_content Repo + uses: actions/checkout@v6 + with: + repository: splunk/security_content + path: security_content + + - name: Install contentctl-ng + run: | + pip install -c security_content/requirements.txt contentctl-ng + - name: Setup AWS CLI and Credentials uses: aws-actions/configure-aws-credentials@v6 with: - aws-access-key-id: ${{ secrets.ACCESS_KEY}} + aws-access-key-id: ${{ secrets.ACCESS_KEY }} aws-secret-access-key: ${{ secrets.SECRET_ACCESS_KEY }} aws-region: us-west-2 - name: Create archive of ONLY the datasets folder + run: | + # Use contentctl-ng to create the Attack Data Archive File + # There is no need to make any other changes to the repo + contentctl-ng build-attack-data-archive + + #This command builds the files: + # - attack_data_archive/attack_data_archive.zip + # - attack_data_archive/metadata.yml + # We intentionally only upload the attack_data_archive.zip file to S3 + ls -lh attack_data_archive + + - name: Upload Attack data archive file to S3 Bucket + run: | + aws s3 cp $NEW_ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/ + + # Backward compatibility: keep publishing the original tar+zstd archive at the original path. + # This MUST run after the contentctl-ng build because it moves the datasets/ folder. + - name: Create legacy tar+zstd archive of ONLY the datasets folder run: | # The structure of the tar + zstd archive should mirror that of checking out the repo directly mkdir attack_data @@ -39,8 +73,8 @@ jobs: # Compress with number of threads equal to number of CPU cores. # Compression level 10 is a great compromise of speed and file size. # File size reductions are diminishing returns after this - determined experimentally. - tar -c attack_data | zstd --compress -T0 -10 -o $ATTACK_DATA_ARCHIVE_FILE + tar -c attack_data | zstd --compress -T0 -10 -o $LEGACY_ATTACK_DATA_ARCHIVE_FILE - - name: Upload Attack data archive file to S3 Bucket + - name: Upload legacy Attack data archive file to S3 Bucket run: | - aws s3 cp $ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/ + aws s3 cp $LEGACY_ATTACK_DATA_ARCHIVE_FILE s3://$BUCKET/