diff --git a/azure-pipelines-templates/run-tests.yml b/azure-pipelines-templates/run-tests.yml index 1288f112..be290641 100644 --- a/azure-pipelines-templates/run-tests.yml +++ b/azure-pipelines-templates/run-tests.yml @@ -102,6 +102,8 @@ jobs: --durations=0 \ --nunit-xml=test-results.xml \ --verbose \ + --reruns 1 \ + --reruns-delay 2 \ env: # Tell Pytest that we're running in a CI environment CI: 1 @@ -123,20 +125,42 @@ jobs: # This will give a user name like 'something macOS 2.7' SG_HUMAN_NAME: $(python_api_human_name) ${{ parameters.os_name }} ${{ parameters.python_version }} SG_HUMAN_PASSWORD: $(python_api_human_password) - # So, first, we need to make sure that two builds running at the same time do not manipulate - # the same entities, so we're sandboxing build nodes based on their name. - SG_PROJECT_NAME: Python API CI - $(Agent.Name) - # The entities created and then reused between tests assume that the same user is always - # manipulating them. Because different builds will be assigned different agents and therefore - # different projects, it means each project needs to have an entity specific to a given user. - # Again, this would have been a lot simpler if we could simply have had a login based on the - # agent name, but alas, the agent name has a space in it which needs to be replaced to something - # else and string substitution can't be made on build variables, only template parameters. - SG_ASSET_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_VERSION_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_SHOT_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_TASK_CONTENT: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} - SG_PLAYLIST_CODE: CI-$(python_api_human_login)-${{ parameters.os_name }}-${{ parameters.python_version }} + # Each job gets its own ephemeral project, scoped to this build + OS + Python version. + # This eliminates state bleed between concurrent runs and across successive builds on the + # same agent. The project is retired in the "Cleanup test project" step below. + SG_PROJECT_NAME: Python API CI - $(Build.BuildId) - ${{ parameters.os_name }} - ${{ parameters.python_version }} + # Entity codes only need to be unique within the project, so fixed strings are fine. + SG_ASSET_CODE: CI-asset + SG_VERSION_CODE: CI-version + SG_SHOT_CODE: CI-shot + SG_TASK_CONTENT: CI-task + SG_PLAYLIST_CODE: CI-playlist + + - task: Bash@3 + displayName: Cleanup test project + condition: always() + inputs: + targetType: inline + script: | + python -c " + import os, shotgun_api3 + sg = shotgun_api3.Shotgun( + os.environ['SG_SERVER_URL'], + os.environ['SG_SCRIPT_NAME'], + os.environ['SG_API_KEY'], + ) + project = sg.find_one('Project', [['name', 'is', os.environ['SG_PROJECT_NAME']]]) + if project: + sg.delete('Project', project['id']) + print('Retired project:', os.environ['SG_PROJECT_NAME']) + else: + print('Project not found, nothing to clean up.') + " + env: + SG_SERVER_URL: $(ci_site) + SG_SCRIPT_NAME: $(ci_site_script_name) + SG_API_KEY: $(ci_site_script_key) + SG_PROJECT_NAME: Python API CI - $(Build.BuildId) - ${{ parameters.os_name }} - ${{ parameters.python_version }} # Explicit call to PublishTestResults@2 and PublishCodeCoverageResults@2 here # instead of relying on pytest-azurepipelines because pytest-azurepipelines diff --git a/tests/base.py b/tests/base.py index eea47fad..fc0c9778 100644 --- a/tests/base.py +++ b/tests/base.py @@ -289,7 +289,7 @@ def _setup_db(cls, config, sg): cls.human_user = _find_or_create_entity(sg, "HumanUser", data) data = {"code": cls.config.asset_code, "project": cls.project} - keys = ["code"] + keys = ["code", "project"] cls.asset = _find_or_create_entity(sg, "Asset", data, keys) data = { diff --git a/tests/requirements.txt b/tests/requirements.txt index 82f6c626..4df3c018 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -11,3 +11,4 @@ pytest pytest-cov pytest-nunit +pytest-rerunfailures diff --git a/tests/test_api.py b/tests/test_api.py index d0e8407e..81dcd3e5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -3329,22 +3329,24 @@ def test_modify_visibility(self): project_1 = projects[0] project_2 = projects[1] + def assert_visibility(project, expected, retries=5, delay=1): + """Poll until schema_field_read reflects the expected visibility value.""" + result = None + for _ in range(retries): + result = self.sg.schema_field_read("Asset", field_name, project)[ + field_name + ]["visible"] + if result == expected: + return + time.sleep(delay) + self.assertEqual(expected, result) + # First, reset the field visibility in a known state, i.e. visible for both projects, # in case the last test run failed midway through. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": True, "editable": True}) self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_2) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_2)[field_name][ - "visible" - ], - ) + assert_visibility(project_2, {"value": True, "editable": True}) # Built-in fields should remain not editable. self.assertFalse( @@ -3360,12 +3362,7 @@ def test_modify_visibility(self): # Hide the field on project 1 self.sg.schema_field_update("Asset", field_name, {"visible": False}, project_1) # It should not be visible anymore. - self.assertEqual( - {"value": False, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": False, "editable": True}) # The field should be visible on the second project. self.assertEqual( @@ -3377,12 +3374,7 @@ def test_modify_visibility(self): # Restore the visibility on the field. self.sg.schema_field_update("Asset", field_name, {"visible": True}, project_1) - self.assertEqual( - {"value": True, "editable": True}, - self.sg.schema_field_read("Asset", field_name, project_1)[field_name][ - "visible" - ], - ) + assert_visibility(project_1, {"value": True, "editable": True}) class TestLibImports(base.LiveTestBase):