update: connectors' documentations with data flows and import graph #1217
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: HrFlow.ai Connectors CI/CD | |
| on: | |
| pull_request_target: | |
| types: [labeled, opened, synchronize, reopened] | |
| branches: | |
| - "master" | |
| push: | |
| branches: | |
| - "master" | |
| env: | |
| POETRY_VERSION: 1.3.1 | |
| BASELINE_PYTHON_VERSION: 3.10.10 | |
| NOX_ENV_DIR: ./.nox | |
| PYTHONPATH: ./src/ | |
| jobs: | |
| core-tests: | |
| runs-on: ubuntu-latest | |
| if: github.event.commits[0].author.name != 'hrflow-semantic-release' | |
| environment: ci | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| lfs: true | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.BASELINE_PYTHON_VERSION }} | |
| - name: Load cached Poetry installation | |
| id: cached-poetry-install | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.local | |
| key: poetry-${{ env.POETRY_VERSION }} | |
| - name: Install poetry | |
| if: steps.cached-poetry-install.outputs.cache-hit != 'true' | |
| run: curl -sSL https://install.python-poetry.org | POETRY_VERSION=$POETRY_VERSION python3 - | |
| - name: Add poetry to PATH | |
| run: export PATH="$HOME/.local/bin:$PATH" | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ env.BASELINE_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install python dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: POETRY_INSTALLER_PARALLEL=1 POETRY_VIRTUALENVS_IN_PROJECT=1 poetry install -E s3 | |
| - name: Run commit hooks | |
| run: poetry run pre-commit run --show-diff-on-failure --hook-stage commit --all-files | |
| - name: Run push hooks | |
| run: poetry run pre-commit run --show-diff-on-failure --hook-stage push --all-files | |
| # This action sets up a Python environment with Nox by: | |
| # - Activating every version of Python that GitHub Actions supports. | |
| # - Installing Nox. | |
| # ==> https://github.com/marketplace/actions/setup-nox | |
| - uses: excitedleigh/[email protected] | |
| - name: Load cached nox venvs | |
| id: cached-nox-venvs | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.NOX_ENV_DIR }} | |
| key: nox-venvs-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-west-1 | |
| - name: Get Connector Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v1 | |
| with: | |
| secret-ids: | | |
| S3_STORE_TEST | |
| parse-json-secrets: true | |
| - name: Check manifest | |
| run: poetry run nox --envdir ${{ env.NOX_ENV_DIR }} -s manifest | |
| - name: Check docs | |
| run: poetry run nox --envdir ${{ env.NOX_ENV_DIR }} -s docs | |
| - name: Run Core tests | |
| run: poetry run nox --envdir ${{ env.NOX_ENV_DIR }} -s tests_s3 | |
| env: | |
| HRFLOW_CONNECTORS_STORE_ENABLED: "1" | |
| HRFLOW_CONNECTORS_LOCALJSON_DIR: "/tmp/" | |
| connectors-integration-tests: | |
| runs-on: ubuntu-latest | |
| if: github.event.commits[0].author.name != 'hrflow-semantic-release' | |
| environment: ci | |
| needs: [core-tests] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| lfs: true | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.BASELINE_PYTHON_VERSION }} | |
| - name: Load cached Poetry installation | |
| id: cached-poetry-install | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.local | |
| key: poetry-${{ env.POETRY_VERSION }} | |
| - name: Add poetry to PATH | |
| run: export PATH="$HOME/.local/bin:$PATH" | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ env.BASELINE_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }} | |
| - uses: excitedleigh/[email protected] | |
| - name: Load cached nox venvs | |
| id: cached-nox-venvs | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.NOX_ENV_DIR }} | |
| key: nox-venvs-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-west-1 | |
| - name: Get Connector Secrets | |
| uses: aws-actions/aws-secretsmanager-get-secrets@v1 | |
| with: | |
| secret-ids: | | |
| S3_STORE_TEST | |
| HRFLOW_CONNECTORS_* | |
| parse-json-secrets: true | |
| - name: Run Connector tests | |
| run: | | |
| poetry run nox -- -s tests -- --no-cov --ignore tests/core --connector=SmartRecruiters --connector=PoleEmploi --connector=Adzuna --connector=Waalaxy --connector=Meteojob --connector=Jobology --connector=Carrevolutis | |
| env: | |
| HRFLOW_CONNECTORS_STORE_ENABLED: "1" | |
| HRFLOW_CONNECTORS_LOCALJSON_DIR: "/tmp/" | |
| cd: | |
| runs-on: ubuntu-latest | |
| if: github.event.commits[0].author.name != 'hrflow-semantic-release' && github.event_name == 'push' && github.ref == 'refs/heads/master' | |
| outputs: | |
| new-release: ${{ steps.semantic-release.outputs.released }} | |
| needs: [core-tests] | |
| environment: release | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| lfs: true | |
| persist-credentials: false | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.BASELINE_PYTHON_VERSION }} | |
| - name: Load cached Poetry installation | |
| id: cached-poetry-install | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.local | |
| key: poetry-${{ env.POETRY_VERSION }} | |
| - name: Add poetry to PATH | |
| run: export PATH="$HOME/.local/bin:$PATH" | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ env.BASELINE_PYTHON_VERSION }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Prepare new release | |
| id: semantic-release | |
| env: | |
| GH_TOKEN: ${{ secrets.PUBLISH_GITHUB_TOKEN }} | |
| run: | | |
| poetry run semantic-release version | |
| poetry run semantic-release publish | |
| - name: Publish to TestPyPI | |
| if: ${{ steps.semantic-release.outputs.released == 'true' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.TEST_PYPI_API_TOKEN }} | |
| repository-url: https://test.pypi.org/legacy/ | |
| - name: Test install from TestPyPI | |
| if: ${{ steps.semantic-release.outputs.released == 'true' }} | |
| run: | | |
| pip install \ | |
| --index-url https://test.pypi.org/simple/ \ | |
| --extra-index-url https://pypi.org/simple \ | |
| hrflow-connectors==${{ steps.semantic-release.outputs.version }} | |
| - name: Publish to PyPI | |
| if: ${{ steps.semantic-release.outputs.released == 'true' }} | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| post-release: | |
| runs-on: ubuntu-latest | |
| needs: [cd] | |
| if: needs.cd.outputs.new-release == 'true' | |
| environment: release | |
| steps: | |
| - name: Dispatch Update Workflows2.0 Environment | |
| uses: peter-evans/[email protected] | |
| with: | |
| token: ${{ secrets.DISPATCH_WORKFLOWS_20_GITHUB_TOKEN }} | |
| repository: ${{ secrets.WORKFLOWS_20_REPO }} | |
| event-type: update-hrflow-connectors |