Entlein/adaptive export config

.bazelrc

@@ -1,10 +1,16 @@
 # Global bazelrc file, see https://docs.bazel.build/versions/master/guide.html#bazelrc.
 
+# Use local Cache directory if building on a VM:
+# On Chef VM, create a directory and comment in the following line:
+# build --disk_cache=</tmp/bazel/> # Optional for multi-user cache: Make this directory owned by a group name e.g. "bazelcache"
 common --noenable_bzlmod --enable_workspace
 
 # Use strict action env to prevent leaks of env vars.
 build --incompatible_strict_action_env
 
+# Use cache
+# build --disk_cache=/tmp/bazel/cache # must not be merged dev only settng
+
 # Only pass through GH_API_KEY for stamped builds.
 # This is still not ideal as it still busts the cache of stamped builds.
 build:stamp --stamp
@@ -174,7 +180,7 @@ build:tsan --define tcmalloc=disabled
 # https://github.com/google/sanitizers/issues/953
 build:tsan --test_env=TSAN_OPTIONS=report_atomic_races=0
 build:tsan --features=tsan
-test:tsan --test_timeout=180,600,1800,3600
+test:tsan --test_timeout=240,600,1800,3600
 
 # Note that we are lumping tests that require root into the BPF tests below
 # to minimize number of configs.

.github/workflows/build_and_test.yaml

@@ -35,32 +35,32 @@ jobs:
     with:
       image-base-name: "dev_image_with_extras"
       ref: ${{ needs.env-protect-setup.outputs.ref }}
-  clang-tidy:
-    runs-on: oracle-vm-16cpu-64gb-x86-64
-    needs: [authorize, env-protect-setup, get-dev-image]
-    container:
-      image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-      with:
-        fetch-depth: 0
-        ref: ${{ needs.env-protect-setup.outputs.ref }}
-    - name: Add pwd to git safe dir
-      run: git config --global --add safe.directory `pwd`
-    - name: get bazel config
-      uses: ./.github/actions/bazelrc
-      with:
-        BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
-    - name: Save Diff Info
-      run: ./ci/save_diff_info.sh
-    - name: Run Clang Tidy
-      shell: bash
-      run: |
-        diff_file="diff_origin_main_cc"
-        if [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ github.event_name }}" == "schedule" ]]; then
-          diff_file="diff_head_cc"
-        fi
-        ./ci/run_clang_tidy.sh -f "${diff_file}"
+  # clang-tidy:
+  #   runs-on: oracle-vm-16cpu-64gb-x86-64
+  #   needs: [authorize, env-protect-setup, get-dev-image]
+  #   container:
+  #     image: ${{ needs.get-dev-image.outputs.image-with-tag }}
+  #   steps:
+  #   - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+  #     with:
+  #       fetch-depth: 0
+  #       ref: ${{ needs.env-protect-setup.outputs.ref }}
+  #   - name: Add pwd to git safe dir
+  #     run: git config --global --add safe.directory `pwd`
+  #   - name: get bazel config
+  #     uses: ./.github/actions/bazelrc
+  #     with:
+  #       BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
+  #   - name: Save Diff Info
+  #     run: ./ci/save_diff_info.sh
+  #   - name: Run Clang Tidy
+  #     shell: bash
+  #     run: |
+  #       diff_file="diff_origin_main_cc"
+  #       if [[ "${{ github.event_name }}" == "push" ]] || [[ "${{ github.event_name }}" == "schedule" ]]; then
+  #         diff_file="diff_head_cc"
+  #       fi
+  #       ./ci/run_clang_tidy.sh -f "${diff_file}"
   code-coverage:
     if: github.event_name == 'push'
     needs: [authorize, env-protect-setup, get-dev-image]
@@ -85,7 +85,7 @@ jobs:
       run: |
         # Github actions container runner creates a docker network without IPv6 support. We enable it manually.
         sysctl -w net.ipv6.conf.lo.disable_ipv6=0
-        ./ci/collect_coverage.sh -u -b main -c "$(git rev-parse HEAD)" -r pixie-io/pixie
+        ./ci/collect_coverage.sh -u -b main -c "$(git rev-parse HEAD)" -r k8sstormcenter/pixie
   generate-matrix:
     needs: [authorize, env-protect-setup, get-dev-image]
     runs-on: oracle-vm-16cpu-64gb-x86-64
@@ -185,19 +185,18 @@ jobs:
       shell: bash
   build-and-test-status:
     if: always()
-    needs: [build-and-test, clang-tidy, generate-matrix]
+    needs: [build-and-test, generate-matrix]
     runs-on: ubuntu-latest
     steps:
-    - if: needs.build-and-test.result == 'success' && needs.clang-tidy.result == 'success'
+    - if: needs.build-and-test.result == 'success'
       run: echo "Build and Test complete ✓"
     - if: >
-        needs.generate-matrix.result == 'success' && needs.clang-tidy.result == 'success'
+        needs.generate-matrix.result == 'success'
         && needs.build-and-test.result == 'skipped'
       run: echo "Build and Test skipped no matrix configs generated ✓"
     - if: >
-        !(needs.build-and-test.result == 'success' && needs.clang-tidy.result == 'success') &&
+        !(needs.build-and-test.result == 'success') &&
         !(needs.generate-matrix.result == 'success' &&
-          needs.clang-tidy.result == 'success' &&
           needs.build-and-test.result == 'skipped')
       run: |
         echo "Build and Test failed"

.github/workflows/cacher.yaml

@@ -12,7 +12,7 @@ jobs:
     with:
       image-base-name: "dev_image"
   populate-caches:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

.github/workflows/cli_release.yaml

@@ -15,12 +15,16 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
+    permissions:
+      contents: read
+      packages: write
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
     env:
       ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
+      MANIFEST_UPDATES: "manifest_updates.json"
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
       with:
@@ -37,23 +41,24 @@ jobs:
         # With some kernel configs (eg. COS), podman only works with legacy iptables.
         update-alternatives --set iptables /usr/sbin/iptables-legacy
         update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
+    - name: Login to GHCR
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
     - name: Import GPG key
       env:
         BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
       run: |
         echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
-    - id: gcloud-creds
-      uses: ./.github/actions/gcloud_creds
-      with:
-        SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }}
     - name: Build & Push Artifacts
       env:
         REF: ${{ github.event.ref }}
         BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         BUILD_NUMBER: ${{ github.run_attempt }}
         JOB_NAME: ${{ github.job }}
-        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
       shell: bash
       run: |
         export TAG_NAME="${REF#*/tags/}"
@@ -66,102 +71,18 @@ jobs:
       with:
         name: linux-artifacts
         path: artifacts/
-    - name: Update GCS Manifest
-      env:
-        ARTIFACT_MANIFEST_BUCKET: "pixie-dev-public"
-        # Use the old style versions file instead of the new updates for the gcs manifest.
-        MANIFEST_UPDATES: ""
-        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
-      run: ./ci/update_artifact_manifest.sh
     - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
       with:
         name: artifact-upload-log
         path: ${{ env.ARTIFACT_UPLOAD_LOG }}
-  sign-release:
-    name: Sign Release for MacOS
-    runs-on: macos-latest
-    needs: build-release
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-      with:
-        fetch-depth: 0
-    - name: Add pwd to git safe dir
-      run: git config --global --add safe.directory `pwd`
-    - name: Install gon
-      run: brew install Bearer/tap/gon
-    - name: Sign CLI release
-      env:
-        REF: ${{ github.event.ref }}
-        AC_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
-        KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
-        CERT_BASE64: ${{ secrets.APPLE_SIGN_CERT_B64 }}
-        CERT_PASSWORD: ${{ secrets.APPLE_SIGN_CERT_PASSWORD }}
-      shell: bash
-      run: |
-        export CERT_PATH="pixie.cert"
-        echo -n "$CERT_BASE64" | base64 --decode -o "$CERT_PATH"
-        export TAG_NAME="${REF#*/tags/}"
-        mkdir -p "artifacts/"
-        export ARTIFACTS_DIR="$(pwd)/artifacts"
-        ./ci/cli_merge_sign.sh
-    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
-      with:
-        name: macos-artifacts
-        path: artifacts/
-  push-signed-artifacts:
-    name: Push Signed Artifacts for MacOS
-    runs-on: ubuntu-latest
-    needs: [get-dev-image, sign-release]
-    container:
-      image: ${{ needs.get-dev-image.outputs.image-with-tag }}
-    env:
-      MANIFEST_UPDATES: "manifest_updates.json"
-    steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
-      with:
-        fetch-depth: 0
-    - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
-      with:
-        name: macos-artifacts
-    - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
-      with:
-        name: artifact-upload-log
-    - name: Import GPG key
-      env:
-        BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
-      run: |
-        echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
-    - id: gcloud-creds
-      uses: ./.github/actions/gcloud_creds
-      with:
-        SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }}
-    - name: Add pwd to git safe dir
-      run: |
-        git config --global --add safe.directory `pwd`
-    - name: Upload signed CLI
-      env:
-        REF: ${{ github.event.ref }}
-        BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
-        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
-        ARTIFACT_UPLOAD_LOG: "artifact_uploads.json"
-      shell: bash
-      run: |
-        export TAG_NAME="${REF#*/tags/}"
-        mkdir -p "artifacts/"
-        export ARTIFACTS_DIR="$(pwd)/artifacts"
-        ./ci/cli_upload_signed.sh
-    - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
-      with:
-        name: macos-signed-artifacts
-        path: artifacts/
     - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
       with:
         name: manifest-updates
         path: ${{ env.MANIFEST_UPDATES }}
   create-github-release:
     name: Create Release on Github
     runs-on: ubuntu-latest
-    needs: push-signed-artifacts
+    needs: build-release
     permissions:
       contents: write
     steps:
@@ -186,9 +107,9 @@ jobs:
         gh release create "${TAG_NAME}" "${prerelease[@]}" \
           --title "CLI ${TAG_NAME#release/cli/}" \
           --notes $'Pixie CLI Release:\n'"${changelog}"
-        gh release upload "${TAG_NAME}" linux-artifacts/* macos-artifacts/*
+        gh release upload "${TAG_NAME}" linux-artifacts/*
   update-gh-artifacts-manifest:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
@@ -217,8 +138,8 @@ jobs:
       env:
         BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
       run: |
-        git config --global user.name 'pixie-io-buildbot'
-        git config --global user.email 'build@pixielabs.ai'
+        git config --global user.name 'k8sstormcenter-buildbot'
+        git config --global user.email 'info@fusioncore.ai'
         git config --global user.signingkey "${BUILDBOT_GPG_KEY_ID}"
         git config --global commit.gpgsign true
     - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0

.github/workflows/cloud_release.yaml

@@ -15,8 +15,11 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
+    permissions:
+      contents: read
+      packages: write
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}
     steps:
@@ -30,15 +33,13 @@ jobs:
       with:
         download_toplevel: 'true'
         BB_API_KEY: ${{ secrets.BB_IO_API_KEY }}
-    - id: gcloud-creds
-      uses: ./.github/actions/gcloud_creds
-      with:
-        SERVICE_ACCOUNT_KEY: ${{ secrets.GH_RELEASE_SA_PEM_B64 }}
     - name: Import GPG key
       env:
         BUILDBOT_GPG_KEY_B64: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
       run: |
         echo "${BUILDBOT_GPG_KEY_B64}" | base64 --decode | gpg --no-tty --batch --import
+    - name: Login to GHCR
+      run: echo "${{ github.token }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
     - name: Build & Push Artifacts
       env:
         REF: ${{ github.event.ref }}
@@ -47,7 +48,6 @@ jobs:
         GH_API_KEY: ${{ secrets.GITHUB_TOKEN }}
         COSIGN_PASSWORD: ${{secrets.COSIGN_PASSWORD}}
         COSIGN_PRIVATE_KEY: ${{secrets.COSIGN_PRIVATE_KEY}}
-        GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.gcloud-creds.outputs.gcloud-creds }}
         BUILDBOT_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
       shell: bash
       run: |

.github/workflows/codeql.yaml

@@ -28,7 +28,7 @@ jobs:
       with:
         category: "/language:go"
   analyze-python:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     permissions:
       actions: read
       contents: read
@@ -42,7 +42,7 @@ jobs:
       with:
         category: "/language:python"
   analyze-javascript:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     permissions:
       actions: read
       contents: read

.github/workflows/copybara_pixie_oss.yaml

@@ -0,0 +1,29 @@
+---
+name: pixie-oss-copybara
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 15 * * *'
+permissions:
+  contents: read
+jobs:
+  run-copybara:
+    runs-on: ubuntu-latest
+    container:
+      # image built from upstream's 9675cc2a commit with ssh added
+      image: ghcr.io/k8sstormcenter/copybara:9675cc2a-ssh
+    steps:
+    - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3  # v3.5.0
+    - id: create-ssh-key
+      env:
+        COPYBARA_SSH_KEY: ${{ secrets.COPYBARA_SSH_KEY }}
+      run: echo "$COPYBARA_SSH_KEY" > /tmp/sshkey && chmod 600 /tmp/sshkey
+    - id: pxapi-copybara
+      env:
+        COPYBARA_GPG_KEY: ${{ secrets.BUILDBOT_GPG_KEY_B64 }}
+        COPYBARA_GPG_KEY_ID: ${{ secrets.BUILDBOT_GPG_KEY_ID }}
+      # 9ae660bce072d1bc1dfbbddada333c88333f4a9a is when fork started
+      # This is only needed for the first copybara run (supplied via --last-rev ${sha} flag)
+      run: >
+        GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /tmp/sshkey'
+        ./ci/run_copybara.sh tools/copybara/upstream_sync/copy.bara.sky