Minor fixes

.github/workflows/test.yml

@@ -0,0 +1,94 @@
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - dev
+
+jobs:
+  test:
+      name: Build and test on Python ${{ matrix.python-version }}
+      runs-on: ubuntu-latest
+
+      strategy:
+        matrix:
+          python-version: ["3.8.12", "3.9.12", "3.10.4"]
+
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v3
+
+        - name: Setup Python ${{ matrix.python-version }}
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python-version }}
+
+        - name: Install dependencies
+          run: |
+            python3 -m pip install --upgrade pip setuptools pytest pytest-cov pytest-subtests pytest-asyncio
+            pip install .
+
+        - name: Check code and formatting
+          run: scripts/check-python.sh .
+
+        - name: Run tests
+          run: python3 -m pytest --cov=. --cov-report=xml --junit-xml pytest.xml --asyncio-mode=strict || true
+
+        - name: Upload results
+          if: always()
+          uses: actions/upload-artifact@v2
+          with:
+            name: Unit test coverage (Python ${{ matrix.python-version }})
+            path: "*.xml"
+
+  sonar:
+    name: Analyse Python project with SonarQube
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .
+          rm -rf build
+
+      - name: Download test results
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - uses: sonarsource/sonarqube-scan-action@master
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+        with:
+          args: >
+            -Dsonar.projectKey=great-ai
+            -Dsonar.login=${{ secrets.SONAR_TOKEN }}
+            -Dsonar.verbose=true
+            -Dsonar.python.coverage.reportPaths=artifacts/*/coverage.xml
+            -Dsonar.coverage.exclusions=**/external/**/*
+            -Dsonar.exclusions=**/external/**/*,artifacts/**/*
+
+  publish-test-results:
+    name: Publish unit test results
+    needs: test
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - name: Publish results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          files: artifacts/*/pytest.xml

.vscode/tasks.json

@@ -2,7 +2,7 @@
   "version": "2.0.0",
   "tasks": [
     {
-      "label": "Format and lint Python",
+      "label": "Format and lint",
       "type": "shell",
       "command": "source .env/bin/activate && scripts/format-python.sh .",
       "windows": {
@@ -18,7 +18,7 @@
       }
     },
     {
-      "label": "Test Python",
+      "label": "Test",
       "type": "shell",
       "command": "source .env/bin/activate && python3 -m pytest .",
       "windows": {

docs/README.md

@@ -3,9 +3,9 @@
 ## Exports
 
 - [clean](src/sus/clean.py)
+- [language](src/sus/clean.py)
 - [unique](src/sus/unique.py)
 - [parallel_map](src/sus/parallel_map.py)
-- [lemmatize](src/sus/lemmatize.py)
 - [evaluate_ranking](src/sus/evaluate_ranking/evaluate_ranking.py)
 - [get_sentences](src/sus/get_sentences.py)
 

setup.cfg

@@ -23,7 +23,7 @@ install_requires =
     unidecode >= 1.3.0
     multiprocess >= 0.70.0.0
     tqdm >= 4.0.0
-    scikit-learn == 1.1.1
+    scikit-learn >= 1.0.0
     matplotlib >= 3.5.0
     numpy >= 1.22.0
     syntok >= 1.4.0

src/great_ai/utilities/__init__.py

@@ -3,7 +3,6 @@ from .config_file import ConfigFile, ParseError
 from .evaluate_ranking import evaluate_ranking
 from .get_sentences import get_sentences
 from .language import english_name_of_language, is_english, predict_language
-from .lemmatize import lemmatize
 from .logger import get_logger
 from .match_names import match_names
 from .parallel_map import parallel_map