ci: Add workflows

.github/CODEOWNERS

@@ -0,0 +1,11 @@
+# See https://help.github.com/articles/about-codeowners/ for syntax
+
+# Core Engineering will be the default owners for everything
+# in the repo. Unless a later match takes precedence,
+# @deepset-ai/core-engineering will be requested for review
+# when someone opens a pull request.
+*                       @deepset-ai/open-source-engineering
+
+# Documentation
+*.md                    @deepset-ai/documentation @deepset-ai/open-source-engineering
+releasenotes/notes/*    @deepset-ai/documentation @deepset-ai/open-source-engineering

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,34 @@
+---
+name: Bug report
+about: Errors you encountered
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**Error message**
+Error that was thrown (if available)
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.
+
+**To Reproduce**
+Steps to reproduce the behavior
+
+**FAQ Check**
+- [ ] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?
+
+**System:**
+ - OS:
+ - GPU/CPU:
+ - Haystack version (commit or version number):
+ - DocumentStore:
+ - Reader:
+ - Retriever:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/actionlint.yml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of string
+  labels: ["cml", "ubuntu-latest-4-cores"]

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'daily'

.github/pull_request_template.md

@@ -0,0 +1,25 @@
+### Related Issues
+
+- fixes #issue-number
+
+### Proposed Changes:
+
+ <!--- In case of a bug: Describe what caused the issue and how you solved it -->
+ <!--- In case of a feature: Describe what did you add and how it works -->
+
+### How did you test it?
+
+<!-- unit tests, integration tests, manual verification, instructions for manual tests -->
+
+### Notes for the reviewer
+
+<!-- E.g. point out section where the reviewer  -->
+
+### Checklist
+
+- I have read the [contributors guidelines](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md) and the [code of conduct](https://github.com/deepset-ai/haystack/blob/main/code_of_conduct.txt)
+- I have updated the related issue with new insights and changes
+- I added unit tests and updated the docstrings
+- I've used one of the [conventional commit types](https://www.conventionalcommits.org/en/v1.0.0/) for my PR title: `fix:`, `feat:`, `build:`, `chore:`, `ci:`, `docs:`, `style:`, `refactor:`, `perf:`, `test:`.
+- I documented my code
+- I ran [pre-commit hooks](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md#installation) and fixed any issue

.github/utils/delete_outdated_docs.py

@@ -0,0 +1,72 @@
+import argparse
+import base64
+import os
+import re
+from pathlib import Path
+from typing import List
+
+import requests
+import yaml
+
+VERSION_VALIDATOR = re.compile(r"^[0-9]+\.[0-9]+$")
+
+
+def readme_token():
+    api_key = os.getenv("README_API_KEY", None)
+    if not api_key:
+        raise Exception("README_API_KEY env var is not set")
+
+    api_key = f"{api_key}:"
+    return base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
+
+
+def create_headers(version: str):
+    return {"authorization": f"Basic {readme_token()}", "x-readme-version": version}
+
+
+def get_docs_in_category(category_slug: str, version: str) -> List[str]:
+    """
+    Returns the slugs of all documents in a category for the specific version.
+    """
+    url = f"https://dash.readme.com/api/v1/categories/{category_slug}/docs"
+    headers = create_headers(version)
+    res = requests.get(url, headers=headers, timeout=10)
+    return [doc["slug"] for doc in res.json()]
+
+
+def delete_doc(slug: str, version: str):
+    url = f"https://dash.readme.com/api/v1/docs/{slug}"
+    headers = create_headers(version)
+    res = requests.delete(url, headers=headers, timeout=10)
+    res.raise_for_status()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Delete outdated documentation from Readme.io. "
+        "It will delete all documents that are not present in the current config files."
+    )
+    parser.add_argument(
+        "-c", "--config-path", help="Path to folder containing YAML documentation configs", required=True, type=Path
+    )
+    parser.add_argument("-v", "--version", help="The version that will have its documents deleted", required=True)
+    args = parser.parse_args()
+
+    configs = [yaml.safe_load(c.read_text()) for c in args.config_path.glob("*.yml")]
+
+    remote_docs = {}
+    for config in configs:
+        category_slug = config["renderer"]["category_slug"]
+        if category_slug in remote_docs:
+            continue
+        docs = get_docs_in_category(category_slug, args.version)
+
+        remote_docs[category_slug] = docs
+
+    for config in configs:
+        doc_slug = config["renderer"]["slug"]
+        category_slug = config["renderer"]["category_slug"]
+        if doc_slug in remote_docs[category_slug]:
+            continue
+
+        delete_doc(doc_slug, args.version)

.github/utils/docstrings_checksum.py

@@ -0,0 +1,49 @@
+from pathlib import Path
+from typing import Iterator
+
+import ast
+import hashlib
+
+
+def docstrings_checksum(python_files: Iterator[Path]):
+    files_content = (f.read_text() for f in python_files)
+    trees = (ast.parse(c) for c in files_content)
+
+    # Get all docstrings from async functions, functions,
+    # classes and modules definitions
+    docstrings = []
+    for tree in trees:
+        for node in ast.walk(tree):
+            if not isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef, ast.ClassDef, ast.Module)):
+                # Skip all node types that can't have docstrings to prevent failures
+                continue
+            docstring = ast.get_docstring(node)
+            if docstring:
+                docstrings.append(docstring)
+
+    # Sort them to be safe, since ast.walk() returns
+    # nodes in no specified order.
+    # See https://docs.python.org/3/library/ast.html#ast.walk
+    docstrings.sort()
+
+    return hashlib.md5(str(docstrings).encode("utf-8")).hexdigest()
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", help="Haystack root folder", required=True, type=Path)
+    args = parser.parse_args()
+
+    # Get all Haystack and rest_api python files
+    root: Path = args.root.absolute()
+    haystack_files = root.glob("haystack/**/*.py")
+    rest_api_files = root.glob("rest_api/**/*.py")
+
+    import itertools
+
+    python_files = itertools.chain(haystack_files, rest_api_files)
+
+    md5 = docstrings_checksum(python_files)
+    print(md5)

.github/utils/generate_openapi_specs.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import json
+from pathlib import Path
+import os
+import sys
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+
+sys.path.append(".")
+from rest_api.utils import get_openapi_specs, get_app, get_pipelines  # pylint: disable=wrong-import-position
+from haystack import __version__  # pylint: disable=wrong-import-position
+
+REST_PATH = Path("./rest_api/rest_api").absolute()
+PIPELINE_PATH = str(REST_PATH / "pipeline" / "pipeline_empty.haystack-pipeline.yml")
+APP_PATH = str(REST_PATH / "application.py")
+
+os.environ["PIPELINE_YAML_PATH"] = PIPELINE_PATH
+
+logging.info("Loading OpenAPI specs from %s with pipeline at %s", APP_PATH, PIPELINE_PATH)
+
+# To initialize the app and the pipelines
+get_app()
+get_pipelines()
+
+# Generate the openapi specs
+specs = get_openapi_specs()
+# Add `x-readme` to disable proxy and limit sample languages on documentation (see https://docs.readme.com/main/docs/openapi-extensions)
+specs.update({"x-readme": {"proxy-enabled": False, "samples-languages": ["curl", "python"]}})
+
+# Dump the specs into a JSON file
+with open("openapi.json", "w") as f:
+    json.dump(specs, f, indent=4)
+    f.write("\n")  # We need to add a newline, otherwise there will be a conflict with end-of-file-fixer pre-commit hook

.github/utils/pydoc-markdown.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e   # Fails on any error in the following loop
+cd docs/pydoc
+rm -rf temp && mkdir temp
+cd temp
+for file in ../config/* ; do
+    echo "Converting $file..."
+    pydoc-markdown "$file"
+done

.github/utils/pyproject_to_requirements.py

@@ -0,0 +1,47 @@
+import argparse
+import re
+import sys
+from pathlib import Path
+
+import toml
+
+matcher = re.compile(r"farm-haystack\[(.+)\]")
+parser = argparse.ArgumentParser(
+    prog="pyproject_to_requirements.py", description="Convert pyproject.toml to requirements.txt"
+)
+parser.add_argument("pyproject_path")
+parser.add_argument("--extra", default="")
+
+
+def resolve(target: str, extras: dict, results: set):
+    if target not in extras:
+        results.add(target)
+        return
+
+    for t in extras[target]:
+        m = matcher.match(t)
+        if m:
+            for i in m.group(1).split(","):
+                resolve(i, extras, results)
+        else:
+            resolve(t, extras, results)
+
+
+def main(pyproject_path: Path, extra: str = ""):
+    content = toml.load(pyproject_path)
+    # basic set of dependencies
+    deps = set(content["project"]["dependencies"])
+
+    if extra:
+        extras = content["project"]["optional-dependencies"]
+        resolve(extra, extras, deps)
+
+    sys.stdout.write("\n".join(sorted(deps)))
+    sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    pyproject_path = Path(args.pyproject_path).absolute()
+
+    main(pyproject_path, args.extra)