Merge pull request #1020 from VisLab/test_branch

Minor updates to test file data

.github/workflows/ci.yaml

@@ -48,92 +48,24 @@ jobs:
           path: ${{ env.pythonLocation }}
           key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }}
 
+      # Install dependencies
       - name: Install dependencies
         run: |
           python -m pip install --upgrade --upgrade-strategy eager pip
-          pip install flake8
-          pip install coverage
           pip install -r requirements.txt
           pip install -r docs/requirements.txt
 
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-
-          flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-
-      - name: Test with unittest
+      # Run spec tests without coverage for non Python 3.9
+      - name: Run spec_test
         env:
           HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          coverage run -m unittest
+        continue-on-error: true
+        run: |  
+           python -m unittest discover spec_tests
 
-      - name: Run spec_test coverage
+      # Run unittest without coverage
+      - name: Test with unittest
         env:
           HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: coverage run --append -m unittest spec_tests/test_errors.py
-        continue-on-error: true
-
-      - name: Archive code coverage results
-        if: ${{matrix.python-version == '3.9'}}
-        uses: actions/upload-artifact@v4
-        with:
-          name: code-coverage-report
-          path: .coverage
-
-  check-secret:
-    runs-on: ubuntu-latest
-    outputs:
-      secrets-exist: ${{ steps.check-for-secrets.outputs.defined }}
-    steps:
-      - name: Check for Secret availability
-        id: check-for-secrets
-        # perform secret check & put boolean result as an output
-        shell: bash
-        run: |
-          if [ "${{ secrets.CC_TEST_REPORTER_ID }}" != '' ]; then
-            echo "defined=true" >> $GITHUB_OUTPUT;
-          else
-            echo "defined=false" >> $GITHUB_OUTPUT;
-          fi
-
-  coverage:
-    name: Publish coverage
-    needs: [build, check-secret]
-    runs-on: ubuntu-latest
-    if: needs.check-secret.outputs.secrets-exist == 'true'
-
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
-
-      - uses: actions/cache@v4
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }}
-
-      - name: Install dependencies
         run: |
-          python -m pip install --upgrade --upgrade-strategy eager pip
-          pip install flake8
-          pip install coverage
-          pip install -r requirements.txt
-          pip install -r docs/requirements.txt
-
-      - name: Download a single artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: code-coverage-report
-
-      - name: publish-coverages
-        with:
-          coverageCommand: coverage xml
-          debug: true
-        uses: paambaati/[email protected]
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-
+          python -m unittest discover tests

.github/workflows/ci_cov.yaml

@@ -0,0 +1,79 @@
+name: CI_COV
+
+on:
+  push:
+    branches: ["*"]
+  pull_request:
+    branches: ["*"]
+
+jobs:
+
+  check-secret:
+    runs-on: ubuntu-latest
+    outputs:
+      secrets-exist: ${{ steps.check-for-secrets.outputs.defined }}
+    steps:
+      - name: Check for Secret availability
+        id: check-for-secrets
+        # perform secret check & put boolean result as an output
+        shell: bash
+        run: |
+          if [ "${{ secrets.CC_TEST_REPORTER_ID }}" != '' ]; then
+            echo "defined=true" >> $GITHUB_OUTPUT;
+          else
+            echo "defined=false" >> $GITHUB_OUTPUT;
+          fi
+
+  build:
+    needs: check-secret
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: [ "3.9" ]
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Install dependencies
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade --upgrade-strategy eager pip
+          pip install flake8 coverage -r requirements.txt -r docs/requirements.txt
+
+      # Run flake8
+      - name: Lint with flake8
+        run: |
+          flake8 . --count --show-source --statistics --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 
+
+      # Run unittest with coverage
+      - name: Test with unittest and coverage
+        env:
+          HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        continue-on-error: true
+        run: |  
+          coverage run -m unittest discover tests
+
+      # Run spec tests with coverage
+      - name: Run spec_test coverage
+        env:
+          HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |  
+          coverage run --append -m unittest discover spec_tests
+          coverage xml
+          ls -la
+
+      # Upload coverage to Code Climate
+      - name: Upload coverage to Code Climate
+        if: needs.check-secret.outputs.secrets-exist == 'true'
+        uses: paambaati/[email protected]
+        env:
+          CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

hed/errors/error_messages.py

@@ -51,16 +51,19 @@ def val_error_element_deprecatedr(tag):
 def val_error_invalid_tag_character(tag, problem_tag):
     return f"Invalid character '{problem_tag}' in tag '{tag}'"
 
+
 @hed_tag_error(ValidationErrors.INVALID_VALUE_CLASS_CHARACTER, has_sub_tag=True,
                actual_code=ValidationErrors.CHARACTER_INVALID)
 def val_error_INVALID_VALUE_CLASS_CHARACTER(tag, problem_tag, value_class):
     return f"Invalid character '{problem_tag}' in tag '{tag}' for value class '{value_class}'"
 
+
 @hed_tag_error(ValidationErrors.INVALID_VALUE_CLASS_VALUE, has_sub_tag=True,
                actual_code=ValidationErrors.VALUE_INVALID)
 def val_error_INVALID_VALUE_CLASS_VALUE(tag, problem_tag, value_class):
     return f"'{tag}' has an invalid value portion for value class '{value_class}'"
 
+
 @hed_error(ValidationErrors.TILDES_UNSUPPORTED)
 def val_error_tildes_not_supported(source_string, char_index):
     character = source_string[char_index]

hed/errors/schema_error_messages.py

@@ -33,19 +33,22 @@ def schema_error_SCHEMA_INVALID_CHILD(tag, child_tag_list):
 
 @hed_error(SchemaAttributeErrors.SCHEMA_ATTRIBUTE_INVALID)
 def schema_error_unknown_attribute(attribute_name, source_tag):
-    return f"Attribute '{attribute_name}' used by '{source_tag}' was not defined in the schema, or was used outside of it's defined class."
+    return f"Attribute '{attribute_name}' used by '{source_tag}' " + \
+    "was not defined in the schema, or was used outside of it's defined class."
 
 
 @hed_error(SchemaWarnings.SCHEMA_PRERELEASE_VERSION_USED, default_severity=ErrorSeverity.WARNING)
 def schema_error_SCHEMA_PRERELEASE_VERSION_USED(current_version, known_versions):
-    return f"Schema version {current_version} used, which is prerelease or unofficial.  Known versions are: {', '.join(known_versions)}"
+    return f"Schema version {current_version} used, which is prerelease or unofficial. " + \
+    f"Known versions are: {', '.join(known_versions)}"
 
 
 @hed_error(SchemaWarnings.SCHEMA_PROLOGUE_CHARACTER_INVALID, default_severity=ErrorSeverity.WARNING,
            actual_code=SchemaWarnings.SCHEMA_CHARACTER_INVALID)
 def schema_error_invalid_character_prologue(char_index, source_string, section_name):
     invalid_char = source_string[char_index]
-    return f"'{section_name}' has invalid character '{invalid_char}' at position {char_index} of string: {source_string}"
+    return f"'{section_name}' has invalid character '{invalid_char}' at " + \
+    f"position {char_index} of string: {source_string}"
 
 
 @hed_error(SchemaWarnings.SCHEMA_INVALID_CHARACTERS_IN_DESC, default_severity=ErrorSeverity.WARNING,
@@ -88,7 +91,7 @@ def schema_error_SCHEMA_CHILD_OF_DEPRECATED(deprecated_tag, non_deprecated_child
 @hed_error(SchemaAttributeErrors.SCHEMA_ATTRIBUTE_VALUE_DEPRECATED,
            actual_code=SchemaAttributeErrors.SCHEMA_DEPRECATION_ERROR)
 def schema_error_SCHEMA_ATTRIBUTE_VALUE_DEPRECATED(tag, deprecated_suggestion, attribute_name):
-    return (f"Tag '{tag}' {attribute_name} uses '{deprecated_suggestion}' which has been deprecated "
+    return (f"Tag '{tag}' {attribute_name} uses '{deprecated_suggestion}' which has been deprecated " + \
             f"and an alternative method of tagging should be used.")
 
 

hed/models/definition_dict.py

@@ -209,7 +209,7 @@ def _find_group(self, definition_tag, group, error_handler):
         # initial validation
         groups = group.groups()
         issues = []
-        tags = group.tags()
+        # tags = group.tags()
         # if len(tags) != 1:
         #     issues += \
         #         ErrorHandler.format_error_with_context(error_handler,

hed/schema/hed_cache.py

@@ -257,7 +257,7 @@ def get_library_data(library_name, cache_folder=None):
             library_data = json.load(file)
         specific_library = library_data[library_name]
         return specific_library
-    except (OSError, CacheException, ValueError, URLError, KeyError) as e:
+    except (OSError, CacheException, ValueError, URLError, KeyError):
         pass
 
     # This failed to get any data for some reason
@@ -288,7 +288,7 @@ def _check_if_url(hed_xml_or_url):
 
 def _create_xml_filename(hed_xml_version, library_name=None, hed_directory=None, prerelease=False):
     """Returns the default file name format for the given version"""
-    prerelease_prefix = f"prerelease/" if prerelease else ""
+    prerelease_prefix = "prerelease/" if prerelease else ""
     if library_name:
         hed_xml_basename = f"{prerelease_prefix}{HED_XML_PREFIX}_{library_name}_{hed_xml_version}{HED_XML_EXTENSION}"
     else:

hed/schema/hed_schema_df_constants.py

@@ -26,7 +26,7 @@
 
 
 DF_EXTRA_SUFFIXES = {PREFIXES_KEY, EXTERNAL_ANNOTATION_KEY}
-DF_SUFFIXES_OMN = { *DF_SUFFIXES, *DF_EXTRA_SUFFIXES}
+DF_SUFFIXES_OMN = {*DF_SUFFIXES, *DF_EXTRA_SUFFIXES}
 
 
 section_mapping_hed_id = {

hed/schema/schema_compliance.py

@@ -209,9 +209,9 @@ def check_duplicate_names(self):
                 error_code = SchemaErrors.SCHEMA_DUPLICATE_NODE
                 if len(values) == 2:
                     error_code = SchemaErrors.SCHEMA_DUPLICATE_FROM_LIBRARY
-                issues_list += self.error_handler.format_error_with_context(error_code, name,
-                                                                            duplicate_tag_list=[entry.name for entry in duplicate_entries],
-                                                                            section=section_key)
+                issues_list += self.error_handler.format_error_with_context(
+                    error_code, name, duplicate_tag_list=[entry.name for entry in duplicate_entries],
+                    section=section_key)
         return issues_list
 
     def check_invalid_chars(self):

hed/schema/schema_io/df2schema.py

@@ -104,7 +104,7 @@ def _read_schema(self, dataframe):
             dataframe (pd.DataFrame): The dataframe for the main tags section
         """
         self._schema._initialize_attributes(HedSectionKey.Tags)
-        known_parent_tags =  {"HedTag": []}
+        known_parent_tags = {"HedTag": []}
         iterations = 0
         # Handle this over multiple iterations in case tags have parent tags listed later in the file.
         # A properly formatted .tsv file will never have parents after the child.
@@ -179,7 +179,7 @@ def _create_tag_entry(self, parent_tags, row_number, row):
                 long_tag_name = tag_name
             return self._create_entry(row_number, row, HedSectionKey.Tags, long_tag_name)
 
-        self._add_fatal_error(row_number, row, f"No tag name found in row.",
+        self._add_fatal_error(row_number, row, "No tag name found in row.",
                               error_code=HedExceptions.GENERIC_ERROR)
 
     def _read_section(self, df, section_key):
@@ -276,5 +276,3 @@ def load_dataframes_from_strings(schema_data):
     return {key: value if isinstance(value, pd.DataFrame) else pd.read_csv(io.StringIO(value), sep="\t",
                                                                            dtype=str, na_filter=False)
             for key, value in schema_data.items()}
-
-

hed/schema/schema_io/df_util.py

@@ -116,7 +116,7 @@ def load_dataframes(filenames, include_prefix_dfs=False):
                                        f"Required column(s) {list(columns_not_in_loaded)} missing from {filename}.  "
                                        f"The required columns are {list(dataframes[key].columns)}", filename=filename)
             dataframes[key] = loaded_dataframe
-        except OSError as e:
+        except OSError:
             # todo: consider if we want to report this error(we probably do)
             pass  # We will use a blank one for this
     return dataframes

hed/schema/schema_io/ontology_util.py

@@ -161,17 +161,19 @@ def _verify_hedid_matches(section, df, unused_tag_ids):
             try:
                 id_int = int(id_value)
                 if id_int not in unused_tag_ids:
-                    hedid_errors += schema_util.format_error(row_number, row,
-                                                             f"'{label}' has id {id_int} which is outside of the valid range for this type.  Valid range is: {min(unused_tag_ids)} to {max(unused_tag_ids)}")
+                    hedid_errors += schema_util.format_error(
+                        row_number, row, f"'{label}' has id {id_int} which is outside " +
+                        f"of the valid range for this type.  Valid range is: " +
+                        f"{min(unused_tag_ids)} to {max(unused_tag_ids)}")
                     continue
             except ValueError:
-                hedid_errors += schema_util.format_error(row_number, row,
-                                                         f"'{label}' has a non-numeric hedID in the dataframe.")
+                hedid_errors += schema_util.format_error(
+                    row_number, row, f"'{label}' has a non-numeric hedID in the dataframe.")
                 continue
 
         if entry_id and entry_id != df_id:
-            hedid_errors += schema_util.format_error(row_number, row,
-                                                     f"'{label}' has hedID '{df_id}' in dataframe, but '{entry_id}' in schema.")
+            hedid_errors += schema_util.format_error(
+                row_number, row, f"'{label}' has hedID '{df_id}' in dataframe, but '{entry_id}' in schema.")
             continue
 
     return hedid_errors
@@ -437,4 +439,3 @@ def _add_annotation_lines(row, annotation_properties, annotation_terms):
 def _get_property_type(row):
     """Gets the property type from the row."""
     return row[constants.property_type] if constants.property_type in row.index else "Class"
-

hed/schema/schema_io/schema2df.py

@@ -362,14 +362,13 @@ def _get_subclass_of(self, tag_entry):
                 return tag_entry.parent.short_tag_name if tag_entry.parent else "HedTag"
 
         base_objects = {
-            HedSectionKey.Units: f"HedUnit",
-            HedSectionKey.UnitClasses: f"HedUnitClass",
-            HedSectionKey.UnitModifiers: f"HedUnitModifier",
-            HedSectionKey.ValueClasses: f"HedValueClass"
+            HedSectionKey.Units: "HedUnit",
+            HedSectionKey.UnitClasses: "HedUnitClass",
+            HedSectionKey.UnitModifiers: "HedUnitModifier",
+            HedSectionKey.ValueClasses: "HedValueClass"
         }
         name, obj_id = self._get_object_name_and_id(base_objects[tag_entry.section_key], include_prefix=True)
 
         if self._get_as_ids:
             return obj_id
         return name
-

hed/schema/schema_io/wiki2schema.py

@@ -512,8 +512,8 @@ def _create_tag_entry(self, parent_tags, row_number, row):
                 long_tag_name = tag_name
             return self._create_entry(row_number, row, HedSectionKey.Tags, long_tag_name)
 
-        self._add_fatal_error(row_number, row, "Schema term is empty or the line is malformed"
-                              , error_code=HedExceptions.WIKI_DELIMITERS_INVALID)
+        self._add_fatal_error(row_number, row, "Schema term is empty or the line is malformed",
+                              error_code=HedExceptions.WIKI_DELIMITERS_INVALID)
 
     def _add_to_dict(self, row_number, row, entry, key_class):
         if entry.has_attribute(HedKey.InLibrary) and not self._loading_merged and not self.appending_to_schema:

hed/scripts/validate_schemas.py

@@ -18,5 +18,6 @@ def main():
         return 1
     return 0
 
+
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())