Merge remote-tracking branch 'upstream/main'

panther-labs · Sep 5, 2023 · 479df65 · 479df65
2 parents 5243f0d + 1963c2e
commit 479df65
Show file tree

Hide file tree

Showing 227 changed files with 5,981 additions and 1,454 deletions.
diff --git a/.ci/opensearch/Dockerfile.opensearch b/.ci/opensearch/Dockerfile.opensearch
@@ -3,5 +3,11 @@ FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
 
 ARG opensearch_path=/usr/share/opensearch
 ARG SECURE_INTEGRATION
+ENV SECURE_INTEGRATION=$SECURE_INTEGRATION
 
 RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then $opensearch_path/bin/opensearch-plugin remove opensearch-security; fi
+
+HEALTHCHECK --start-period=20s --interval=30s \
+  CMD curl -sf -retry 5 --max-time 5 --retry-delay 5 --retry-max-time 30 \
+  $(if $SECURE_INTEGRATION; then echo "-u admin:admin -k https://"; fi)"localhost:9200" \
+  || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'
diff --git a/.ci/opensearch/docker-compose.yml b/.ci/opensearch/docker-compose.yml
@@ -1,8 +1,10 @@
 version: '3'
 
 services:
-
   opensearch:
+    deploy:
+      restart_policy:
+        condition: any
     build:
       context: .
       dockerfile: Dockerfile.opensearch
@@ -12,6 +14,7 @@ services:
     environment:
       - discovery.type=single-node
       - bootstrap.memory_lock=true
+      - path.repo=/usr/share/opensearch/mnt
     ports:
       - "9200:9200"
     user: opensearch
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 # This should match the team set up in https://github.com/orgs/opensearch-project/teams and include any additional contributors
-*   @dblock @VijayanB @svencowart @VachaShah
+*   @dblock @VijayanB @VachaShah @Jakob3xD
diff --git a/.github/workflows/dependabot_pr.yml b/.github/workflows/dependabot_pr.yml
@@ -33,6 +33,12 @@ jobs:
         uses: dangoslen/dependabot-changelog-helper@v2
         with:
           version: 'Unreleased'
+
+      - name: Install dependencies
+        run: npm i -g npm@8 && npm i -g prettier
+
+      - name: Format markdown files with prettier
+        run: prettier --prose-wrap never --write **/*.md
 
       - name: Commit the changes
         uses: stefanzweifel/git-auto-commit-action@v4

diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Check license headers
         run: |

diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: lychee Link Checker
         id: lychee
         uses: lycheeverse/[email protected]

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -16,15 +16,24 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
+        with: { go-version: '1.x' }
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
         with:
-          go-version: 1.17
+          version: v1.53.3
+          args: -v -c .golangci.yml
+          only-new-issues: true
+
+  prettify:
+    name: Prettify
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with: { fetch-depth: 1 }
       - name: Install dependencies
-        run: go get -u golang.org/x/lint/golint
-        env:
-          GOBIN: ${{ env.GOROOT }}/bin
-        shell: bash
-      - run: go version
-      - run: make lint
+        run: npm i -g npm@8 && npm i -g prettier
+      - name: Check markdown files
+        run: prettier --prose-wrap never --check **/*.md
diff --git a/.github/workflows/test-compatibility.yml b/.github/workflows/test-compatibility.yml
@@ -11,25 +11,22 @@ jobs:
         secured: ["true", "false"]
         entry:
           - { opensearch_version: 1.1.0 }
-          - { opensearch_version: 1.2.0 }
-          - { opensearch_version: 1.2.1 }
-          - { opensearch_version: 1.2.2 }
-          - { opensearch_version: 1.2.3 }
           - { opensearch_version: 1.2.4 }
-          - { opensearch_version: 1.3.0 }
-          - { opensearch_version: 1.3.1 }
-          - { opensearch_version: 1.3.2 }
-          - { opensearch_version: 1.3.3 }
-          - { opensearch_version: 1.3.4 }
-          - { opensearch_version: 2.0.0 }
+          - { opensearch_version: 1.3.8 }
           - { opensearch_version: 2.0.1 }
           - { opensearch_version: 2.1.0 }
-          - { opensearch_version: 2.2.0 }
+          - { opensearch_version: 2.2.1 }
+          - { opensearch_version: 2.3.0 }
+          - { opensearch_version: 2.4.1 }
+          - { opensearch_version: 2.5.0 }
+          - { opensearch_version: 2.6.0 }
+          - { opensearch_version: 2.7.0 }
+          - { opensearch_version: 2.8.0 }
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
 
-      - uses: actions/setup-go@v2.1.3
+      - uses: actions/setup-go@v3
         with: { go-version: '1.x' }
 
       - run: go version
@@ -46,10 +43,14 @@ jobs:
           export OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }}
           export SECURE_INTEGRATION=${{ matrix.secured }}
           make cluster.clean cluster.build cluster.start
+          for attempt in `seq 25`; do sleep 5; \
+          if curl -s $(if $SECURE_INTEGRATION; then echo "-ku admin:admin https://"; fi)localhost:9200; \
+          then echo '=====> ready'; break; fi; if [ $attempt == 25 ]; then exit 1; fi; echo '=====> waiting...'; done
 
       - name: Integration test without security
         if: ${{ matrix.secured == 'false'}}
-        run: make test-integ race=true
+        run: |
+          make test-integ race=true
 
       - name: Integration test with security
         if: ${{ matrix.secured == 'true'}}

diff --git a/.github/workflows/test-integration-unreleased.yml b/.github/workflows/test-integration-unreleased.yml
@@ -14,43 +14,63 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        entry:
-          - { opensearch_ref: '1.x' }
-          - { opensearch_ref: '2.0' }
-          - { opensearch_ref: '2.x' }
-          - { opensearch_ref: 'main' }
+        opensearch_ref: [ '1.x', '2.x', 'main' ]
     steps:
       - name: Checkout OpenSearch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: opensearch-project/OpenSearch
           ref: ${{ matrix.entry.opensearch_ref }}
           path: opensearch
 
+      - name: Get OpenSearch branch top
+        id: get-key
+        working-directory: opensearch
+        run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT
+
+      - name: Restore cached build
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: opensearch/distribution/archives/linux-tar/build/distributions
+          key: ${{ steps.get-key.outputs.key }}
+
       - name: Assemble OpenSearch
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        working-directory: opensearch
+        run: ./gradlew :distribution:archives:linux-tar:assemble
+
+      - name: Save cached build
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: opensearch/distribution/archives/linux-tar/build/distributions
+          key: ${{ steps.get-key.outputs.key }}
+
+      - name: Run OpenSearch
+        working-directory: opensearch/distribution/archives/linux-tar/build/distributions
         run: |
-          cd opensearch
-          ./gradlew assemble
-        # This step runs the docker image generated during gradle assemble in OpenSearch. It is tagged as opensearch:test.
-        # Reference: https://github.com/opensearch-project/OpenSearch/blob/2.0/distribution/docker/build.gradle#L190
-      - name: Run Docker Image
-        run: |
-          docker run -p 9200:9200 -p 9600:9600 -d -e "discovery.type=single-node" -e "bootstrap.memory_lock=true" opensearch:test
-          sleep 90
+          tar xf opensearch-min-*
+          ./opensearch-*/bin/opensearch -Epath.repo=/usr/share/opensearch/mnt &
+          for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done
       
       - name: Checkout Go Client
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          path: go-client
 
       - name: Setup Go
-        uses: actions/setup-go@v2.1.3
+        uses: actions/setup-go@v3
         with: { go-version: '1.x' }
 
-      - name: Increase system limits
-        run: |
-          sudo swapoff -a
-          sudo sysctl -w vm.swappiness=1
-          sudo sysctl -w fs.file-max=262144
-          sudo sysctl -w vm.max_map_count=262144
-
       - name: Integration test
+        working-directory: go-client
         run: make test-integ race=true
+
+      - name: Save server logs
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: opensearch-logs-${{ matrix.opensearch_ref }}-go-${{ matrix.python-version }}
+          path: |
+            opensearch/distribution/archives/linux-tar/build/distributions/**/logs/*
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -16,9 +16,9 @@ jobs:
     name: Tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
-      - uses: actions/setup-go@v2.1.3
+      - uses: actions/setup-go@v3
         with: { go-version: '1.x' }
       - run: go version
       - name: Increase system limits
@@ -43,9 +43,9 @@ jobs:
     env:
       SECURE_INTEGRATION: true
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
-      - uses: actions/setup-go@v2.1.3
+      - uses: actions/setup-go@v3
         with: { go-version: '1.x' }
       - run: go version
       - name: Increase system limits

diff --git a/.github/workflows/test-unit-tip.yml b/.github/workflows/test-unit-tip.yml
@@ -35,7 +35,7 @@ jobs:
           bash make.bash
           echo "GOROOT=$GOROOT" >> $GITHUB_ENV
           echo "$GOROOT/bin" >> $GITHUB_PATH
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
       - run: go version
       - run: make test-unit race=true
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
@@ -21,9 +21,9 @@ jobs:
         go: ['1.x']
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
-      - uses: actions/setup-go@v2.1.3
+      - uses: actions/setup-go@v3
         with: { go-version: "${{ matrix.go }}" }
       - run: go version
       - run: make test-unit race=true
@@ -37,9 +37,9 @@ jobs:
     name: Benchmarks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with: { fetch-depth: 1 }
-      - uses: actions/setup-go@v2.1.3
+      - uses: actions/setup-go@v3
         with: { go-version: '1.x' }
       - run: go version
       - run: make test-bench
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,5 @@ tmp/
 .idea
 .vscode
 bin/
+
+.DS_Store
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,3 +5,5 @@ tmp/ @@
     .idea
     .vscode
     bin/
+    .DS_Store