diff --git a/core/collections/constants.py b/core/collections/constants.py index 8048f2d8f..3bf1acf25 100644 --- a/core/collections/constants.py +++ b/core/collections/constants.py @@ -4,7 +4,6 @@ EXPRESSION_NUMBER_OF_PARTS_WITH_VERSION = 9 INCLUDE_REFERENCES_PARAM = 'includeReferences' -REFERENCE_ALREADY_EXISTS = 'Concept or Mapping reference name must be unique in a collection.' CONCEPT_FULLY_SPECIFIED_NAME_UNIQUE_PER_COLLECTION_AND_LOCALE = "Concept fully specified name must be unique for " \ "same collection and locale." CONCEPT_PREFERRED_NAME_UNIQUE_PER_COLLECTION_AND_LOCALE = "Concept preferred name must be unique for same collection " \ diff --git a/core/collections/models.py b/core/collections/models.py index c8b41f2fb..14e365ffc 100644 --- a/core/collections/models.py +++ b/core/collections/models.py @@ -12,7 +12,7 @@ from pydash import get, compact from core.collections.constants import ( - COLLECTION_TYPE, REFERENCE_ALREADY_EXISTS, CONCEPT_FULLY_SPECIFIED_NAME_UNIQUE_PER_COLLECTION_AND_LOCALE, + COLLECTION_TYPE, CONCEPT_FULLY_SPECIFIED_NAME_UNIQUE_PER_COLLECTION_AND_LOCALE, CONCEPT_PREFERRED_NAME_UNIQUE_PER_COLLECTION_AND_LOCALE, COLLECTION_VERSION_TYPE, REFERENCE_TYPE_CHOICES, CONCEPT_REFERENCE_TYPE, MAPPING_REFERENCE_TYPE, SOURCE_MAPPINGS, SOURCE_TO_CONCEPTS, TRANSFORM_TO_RESOURCE_VERSIONS, COLLECTION_REFERENCE_TYPE, TRANSFORM_TO_EXTENSIONAL) @@ -175,18 +175,7 @@ def validate(self, reference): else: reference.last_resolved_at = None errors = {} - if reference.expression: - same_refs = self.references.filter(expression=reference.expression, include=reference.include) - if same_refs.exists(): - errors = { - reference.expression: { - 'errors': [{ - 'description': REFERENCE_ALREADY_EXISTS, - 'conflicting_references': [ref.uri for ref in same_refs] - }] - } - } - return errors + if self.is_openmrs_schema and self.expansion_uri: if reference._concepts is None or reference._concepts.count() == 0: # pylint: disable=protected-access return None @@ -219,7 +208,7 @@ def check_concept_uniqueness_in_collection_and_locale_by_name_attribute( matching_names_in_concept[name_key] = True other = other_concepts_in_collection.filter( names__name=name.name, names__locale=name.locale, **{f"names__{attribute}": value}) - if other.exists(): + if other.exclude(versioned_object_id=concept.versioned_object_id).exists(): for other_concept in other: conflicting_concept_name = other_concept.names.filter( name=name.name, locale=name.locale, **{attribute: value}).first() diff --git a/core/collections/parsers.py b/core/collections/parsers.py index 580b33851..63f93dbf4 100644 --- a/core/collections/parsers.py +++ b/core/collections/parsers.py @@ -83,7 +83,7 @@ def to_objects(self): from core.collections.models import CollectionReference cascade_to_concepts = self.cascade == SOURCE_TO_CONCEPTS cascade_mappings = self.cascade == SOURCE_MAPPINGS - should_cascade_now = cascade_mappings or cascade_to_concepts + should_cascade_now = (cascade_mappings or cascade_to_concepts) and self.transform for reference in self.references: reference['valueset'] = self.get_formatted_valueset(reference.get('valueset')) collection_reference = CollectionReference(**reference) diff --git a/core/collections/tests/tests.py b/core/collections/tests/tests.py index caa3fbe76..e9a5afb39 100644 --- a/core/collections/tests/tests.py +++ b/core/collections/tests/tests.py @@ -88,24 +88,17 @@ def test_add_expressions(self): self.assertEqual(collection.references.first().expression, concept.uri) self.assertEqual(collection.expansion.concepts.first().id, concept.id) self.assertEqual(collection.active_concepts, 1) + self.assertEqual(concept.references.count(), 1) - _, errors = collection.add_expressions({'concepts': [concept.uri]}, collection.created_by) - self.assertEqual( - errors, { - concept.uri: { - concept.uri: { - 'errors': [{ - 'description': 'Concept or Mapping reference name must be unique in a collection.', - 'conflicting_references': [collection.references.first().uri] - }] - } - } - } - ) + added, errors = collection.add_expressions({'concepts': [concept.uri]}, collection.created_by) + self.assertEqual(errors, {}) + self.assertIsNotNone(added[0].uri) + self.assertEqual(added[0].concepts.first().uri, concept.uri) collection.refresh_from_db() self.assertEqual(collection.expansion.concepts.count(), 1) - self.assertEqual(collection.references.count(), 1) + self.assertEqual(collection.references.count(), 2) self.assertEqual(collection.active_concepts, 1) + self.assertEqual(concept.references.count(), 2) def test_add_expressions_openmrs_schema(self): collection = OrganizationCollectionFactory(custom_validation_schema=OPENMRS_VALIDATION_SCHEMA) @@ -191,33 +184,6 @@ def test_seed_references(self): self.assertEqual(collection1.references.first().expression, collection2.references.first().expression) self.assertNotEqual(collection1.references.first().id, collection2.references.first().id) - def test_validate_reference_already_exists(self): - collection = OrganizationCollectionFactory() - expansion = ExpansionFactory(collection_version=collection) - collection.expansion_uri = expansion.uri - collection.save() - ch_locale = ConceptNameFactory.build(locale_preferred=True, locale='ch') - en_locale = ConceptNameFactory.build(locale_preferred=True, locale='en') - concept = ConceptFactory(names=[ch_locale, en_locale]) - reference = CollectionReference(expression=concept.uri, collection=collection) - reference.save() - - self.assertEqual(collection.references.count(), 1) - - errors = collection.validate(reference) - - self.assertEqual( - errors, - { - concept.uri: { - 'errors': [{ - 'description': 'Concept or Mapping reference name must be unique in a collection.', - 'conflicting_references': [reference.uri] - }] - } - } - ) - def test_validate_openmrs_schema_duplicate_locale_type(self): ch_locale = ConceptNameFactory.build(locale_preferred=True, locale='ch') en_locale = ConceptNameFactory.build(locale_preferred=True, locale='en') @@ -1081,7 +1047,7 @@ def test_add_references_task(self): 'sourcemappings' ) - self.assertEqual(len(added_references), 4) + self.assertEqual(len(added_references), 3) self.assertEqual(errors, {}) self.assertListEqual( sorted(list( @@ -1089,9 +1055,10 @@ def test_add_references_task(self): )), sorted([ concept1.get_latest_version().url, concept2.get_latest_version().url, - mapping1.url, mapping2.get_latest_version().url, + mapping2.get_latest_version().url, ]) ) + self.assertEqual(mapping1.references.count(), 1) self.assertEqual( sorted(list(expansion.concepts.values_list('uri', flat=True))), sorted([concept1.get_latest_version().url, concept2.get_latest_version().url]) @@ -1130,7 +1097,6 @@ def test_add_references_task(self): sorted([ concept1.get_latest_version().url, concept2.get_latest_version().url, - mapping1.url, mapping2.get_latest_version().url, 'http://foo-system.com/concepts/bar/', 'http://foo-system2.com|v1/concepts/bar/' diff --git a/core/common/constants.py b/core/common/constants.py index fe86237a0..6a7a44d29 100644 --- a/core/common/constants.py +++ b/core/common/constants.py @@ -128,7 +128,7 @@ CASCADE_DIRECTION_PARAM = 'reverse' OMIT_IF_EXISTS_IN = 'omitIfExistsIn' INCLUDE_SELF = 'includeSelf' -FACET_SIZE = 20 +FACET_SIZE = 100 ALL = '*' CANONICAL_URL_REQUEST_PARAM = 'canonicalUrl' SAME_STANDARD_CHECKSUM_ERROR = 'No changes detected. Standard checksum is same as last version.' diff --git a/core/common/mixins.py b/core/common/mixins.py index 085b41256..fc76cce39 100644 --- a/core/common/mixins.py +++ b/core/common/mixins.py @@ -29,7 +29,7 @@ from .exceptions import Http403 from .utils import write_csv_to_s3, get_csv_from_s3, get_query_params_from_url_string, compact_dict_by_values, \ to_owner_uri, parse_updated_since_param, get_export_service, to_int, get_truthy_values, generate_temp_version, \ - canonical_url_to_url_and_version, decode_string + canonical_url_to_url_and_version, decode_string, get_falsy_values from ..concepts.constants import PERSIST_CLONE_ERROR from ..toggles.models import Toggle @@ -164,12 +164,6 @@ def __get_cached_data_if_any(self, request): params.pop('brief', None) query_string = urlencode(params, doseq=True) parent = self.parent_resource - repo_default_filter = get(parent, 'concept_filter_default') - if repo_default_filter: - query_string += '&' + urlencode({ - k: json.dumps(v) if isinstance(v, (dict, list)) else v - for k, v in repo_default_filter.items() - }) key_body, key_headers = parent.get_concepts_cache_keys() if '/concepts' in base_path else ( parent.get_mappings_cache_keys()) @@ -580,32 +574,21 @@ def collections_url(self): return self.uri + 'collections/' def get_repo_events_criteria(self, private=False): - criteria = None sources = self.source_set.filter(is_active=True) collections = self.collection_set.filter(is_active=True) if not private: - sources = self.source_set.filter(public_access__in=[ACCESS_TYPE_VIEW, ACCESS_TYPE_EDIT]) - collections = self.collection_set.filter(public_access__in=[ACCESS_TYPE_VIEW, ACCESS_TYPE_EDIT]) + sources = sources.filter(public_access__in=[ACCESS_TYPE_VIEW, ACCESS_TYPE_EDIT]) + collections = collections.filter(public_access__in=[ACCESS_TYPE_VIEW, ACCESS_TYPE_EDIT]) - for source in sources: - if criteria is None: - criteria = Q(referenced_object_url=source.uri) - else: - criteria |= Q(referenced_object_url=source.uri) - for collection in collections: - if criteria is None: - criteria = Q(referenced_object_url=collection.uri) - else: - criteria |= Q(referenced_object_url=collection.uri) + source_uris = sources.values_list('uri', flat=True) + collection_uris = collections.values_list('uri', flat=True) - return criteria + return Q(referenced_object_url__in=[*source_uris, *collection_uris]) def get_repo_events(self, private=False): from core.events.models import Event criteria = self.get_repo_events_criteria(private) - if criteria is None: - return Event.objects.none() queryset = Event.objects.filter(criteria) return queryset if private else queryset.filter(public=True) diff --git a/core/common/models.py b/core/common/models.py index cc0323f8a..28beeb92f 100644 --- a/core/common/models.py +++ b/core/common/models.py @@ -24,6 +24,7 @@ canonical_url_to_url_and_version, get_current_authorized_user, encode_string, decode_string from core.common.utils import to_owner_uri from core.settings import DEFAULT_LOCALE +from . import ERRBIT_LOGGER from .checksums import ChecksumModel from .constants import ( ACCESS_TYPE_CHOICES, DEFAULT_ACCESS_TYPE, NAMESPACE_REGEX, @@ -1105,6 +1106,11 @@ def retired_mappings_count(self): @property def concepts_distribution(self): facets = self.get_concept_facets() + print("Concept facets Raw", facets) + try: + print("Concept facets", facets.to_dict()) + except: # pylint: disable=bare-except + pass return { 'active': self.active_concepts, 'retired': self.retired_concepts_count, @@ -1185,6 +1191,7 @@ def _get_resource_facets(self, facet_class, filters=None, **kwargs): try: facets = search.execute().facets except TransportError as ex: # pragma: no cover + ERRBIT_LOGGER.log(ex) raise Http400(detail=get(ex, 'info') or get(ex, 'error') or str(ex)) from ex return facets diff --git a/core/common/permissions.py b/core/common/permissions.py index e5a0a194b..022a3a301 100644 --- a/core/common/permissions.py +++ b/core/common/permissions.py @@ -1,6 +1,7 @@ from rest_framework.permissions import BasePermission from core.common.constants import ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW +from core.users.constants import MAPPER_AI_ASSISTANT_GROUP class HasPrivateAccess(BasePermission): @@ -95,3 +96,11 @@ def has_object_permission(self, request, view, obj): if obj.public_access in [ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW]: return True return super().has_object_permission(request, view, obj) + +class IsInAuthGroup(BasePermission): + """ + The user belongs to one of the authorized groups + """ + def has_permission(self, request, view): + user = request.user + return user.is_authenticated and user.has_auth_group(MAPPER_AI_ASSISTANT_GROUP) diff --git a/core/common/search.py b/core/common/search.py index e7863eba0..427ea0698 100644 --- a/core/common/search.py +++ b/core/common/search.py @@ -5,6 +5,7 @@ from elasticsearch_dsl import FacetedSearch, Q from pydash import compact, get +from core.common.constants import ES_REQUEST_TIMEOUT from core.common.utils import is_url_encoded_string @@ -134,20 +135,20 @@ def get_exact_match_criterion( @staticmethod def get_match_phrase_criteria(field, search_str, boost): - criteria = CustomESSearch.get_term_match_criteria(field, search_str, boost) - if field == 'external_id': - return criteria - return criteria | CustomESSearch.get_prefix_criteria( - field, search_str, boost - ) | Q('match_phrase', **{field: {'query': search_str, 'boost': boost}}) + if field in ['external_id', '_name', '_synonyms', 'repo_owner'] or field.startswith('_'): + return CustomESSearch.get_term_match_criteria(field, search_str, boost) + + return Q( + 'match_phrase', **{field: {'query': search_str.lower(), 'boost': boost + 75}} + ) | CustomESSearch.get_prefix_criteria(field, search_str.lower(), boost + 50) @staticmethod def get_term_match_criteria(field, search_str, boost): - return Q('term', **{field: {'value': search_str, 'boost': boost + 100}}) + return Q('term', **{field: {'value': search_str, 'boost': boost + 100, 'case_insensitive': True}}) @staticmethod def get_prefix_criteria(field, search_str, boost): - return Q('prefix', **{field: {'value': search_str, 'boost': boost + 95}}) + return Q('match_phrase_prefix', **{field: {'query': search_str.lower(), 'boost': boost, "max_expansions": 20}}) @staticmethod def get_match_criteria(field, search_str, boost): @@ -202,14 +203,18 @@ def apply_aggregation_score_histogram(self): def apply_aggregation_score_stats(self): self._dsl_search.aggs.bucket("score", "stats", script="_score") - def to_queryset(self, keep_order=True, normalized_score=False): + def to_queryset(self, keep_order=True, normalized_score=False, exact_count=True): # pylint:disable=too-many-locals """ This method return a django queryset from the an elasticsearch result. It cost a query to the sql db. """ - s, hits = self.__get_response() + import time + start_time = time.time() + s, hits, total = self.__get_response(exact_count) + print("ES query execute", time.time() - start_time) max_score = hits.max_score or 1 + start_time = time.time() for result in hits.hits: _id = get(result, '_id') self.scores[int(_id)] = { @@ -219,6 +224,7 @@ def to_queryset(self, keep_order=True, normalized_score=False): highlight = get(result, 'highlight') if highlight: self.highlights[int(_id)] = highlight.to_dict() + print("Highlights/Score", time.time() - start_time) if self.document and self.document.__name__ == 'RepoDocument': from core.sources.models import Source from core.collections.models import Collection @@ -240,15 +246,16 @@ def to_queryset(self, keep_order=True, normalized_score=False): ) qs = qs.order_by(preserved_order) self.queryset = qs - self.total = hits.total.value + self.total = total or 0 def get_aggregations(self, verbose=False, raw=False): - s, _ = self.__get_response() + s, _, total = self.__get_response() result = s.aggs.to_dict() if raw: return result self.max_score = result['score']['max'] + self.total = total or 0 return self._get_score_buckets( self.max_score, result['distribution']['buckets'], verbose) @@ -301,13 +308,18 @@ def append_to_bucket(_bucket, _score, count): return [build_confidence(high), build_confidence(medium), build_confidence(low)] - def __get_response(self): + def __get_response(self, exact_count=True): # Do not query again if the es result is already cached + total = None if not hasattr(self._dsl_search, '_response'): # We only need the meta fields with the models ids - s = self._dsl_search.source(excludes=['*']) + s = self._dsl_search.source(False) + s = s.params(request_timeout=ES_REQUEST_TIMEOUT) + if exact_count: + total = s.count() + s = s.params(track_total_hits=False, request_cache=True) s = s.execute() hits = s.hits self.max_score = hits.max_score - return s, hits - return self._dsl_search, None + return s, hits, total + return self._dsl_search, None, total diff --git a/core/common/utils.py b/core/common/utils.py index a59e6e8c7..da859c549 100644 --- a/core/common/utils.py +++ b/core/common/utils.py @@ -496,6 +496,8 @@ def is_versioned_uri(expression): def to_parent_uri(expression): + if not expression: + return expression splitter = None if '/concepts/' in expression: splitter = '/concepts/' diff --git a/core/common/views.py b/core/common/views.py index efeea7e2d..88e4aaa73 100644 --- a/core/common/views.py +++ b/core/common/views.py @@ -334,7 +334,7 @@ def get_query_criteria(attr, val): return Q( "nested", path="mapped_codes", - query=Q("match", **{f"mapped_codes.{path}": val}) + query=Q("term", **{f"mapped_codes.{path}": val}) ) if is_property: property_code = attr.split('properties__', 1)[1] @@ -344,7 +344,7 @@ def get_query_criteria(attr, val): return ~Q("exists", field=new_attr) return Q('term', **{f"properties.{property_code}.keyword": val.strip('\"').strip('\'')}) - return Q('match', **{attr: val.strip('\"').strip('\'')}) + return Q('term', **{attr: val.strip('\"').strip('\'')}) def get_query(attr, val): """ @@ -545,16 +545,17 @@ def get_facets(self): if self.should_search_latest_repo() and self.is_source_child_document_model() and 'source_version' in facets: facets['source_version'] = [facet for facet in facets['source_version'] if facet[0] != 'HEAD'] is_global_scope = ('org' not in self.kwargs and 'user' not in self.kwargs and not self.user_is_self) + + facets.pop('collection_owner_url', None) if is_global_scope: - facets.pop('source_version', None) facets.pop('collection_version', None) facets.pop('expansion', None) - facets.pop('collection_owner_url', None) else: facets.pop('owner', None) - facets.pop('owner_type', None) + facets.pop('ownerType', None) if 'source' in self.kwargs: facets.pop('source', None) + facets.pop('source_version', None) elif 'collection' in self.kwargs: facets.pop('collection', None) facets.pop('collection_owner_url', None) @@ -649,16 +650,16 @@ def is_user_scope(self): return False def get_public_criteria(self): - criteria = Q('match', public_can_view=True) + criteria = Q('term', public_can_view=True) user = self.request.user if user.is_authenticated: username = user.username from core.orgs.documents import OrganizationDocument if self.document_model in [OrganizationDocument]: - criteria |= (Q('match', public_can_view=False) & Q('match', user=username)) + criteria |= (Q('term', public_can_view=False) & Q('term', user=username)) if self.is_concept_container_document_model() or self.is_source_child_document_model(): - criteria |= (Q('match', public_can_view=False) & Q('match', created_by=username)) + criteria |= (Q('term', public_can_view=False) & Q('term', created_by=username)) return criteria @@ -690,9 +691,9 @@ def __apply_common_search_filters(self, ignore_retired_filter=False, force=False updated_by = self.request.query_params.get(UPDATED_BY_USERNAME_PARAM, None) if updated_by: - results = results.query("terms", updated_by=compact(updated_by.split(','))) + results = results.filter("terms", updated_by=compact(updated_by.split(','))) if self.is_canonical_specified(): - results = results.query( + results = results.filter( 'match_phrase', _canonical_url=format_url_for_search(self.request.query_params.get(CANONICAL_URL_REQUEST_PARAM)) ) @@ -702,25 +703,21 @@ def __apply_common_search_filters(self, ignore_retired_filter=False, force=False default_filters[latest_attr] = True for field, value in default_filters.items(): - results = results.query("match", **{field: value}) + results = results.filter("term", **{field: value}) updated_since = parse_updated_since_param(self.request.query_params) if updated_since: - results = results.query('range', last_update={"gte": updated_since}) + results = results.filter('range', last_update={"gte": updated_since}) if not ignore_retired_filter and self._should_exclude_retired_from_search_results(): - results = results.query('match', retired=False) + results = results.filter('term', retired=False) include_private = self._should_include_private() if not include_private: - results = results.query(self.get_public_criteria()) - faceted_criterion = self.get_faceted_criterion( - repo_default_filters=get( - self, 'parent_resource.concept_filter_default' - ) if apply_default_filters and self.is_concept_document() else None - ) + results = results.filter(self.get_public_criteria()) + faceted_criterion = self.get_faceted_criterion() if faceted_criterion: - results = results.query(faceted_criterion) + results = results.filter(faceted_criterion) return results def is_canonical_specified(self): @@ -866,18 +863,49 @@ def __get_search_results(self, ignore_retired_filter=False, sort=True, highlight for key, value in kwargs_filters.items(): attr = to_snake_case(key) if isinstance(value, list): - criteria = Q('match', **{attr: get(value, '0')}) + criteria = Q('term', **{attr: get(value, '0')}) for val in value[1:]: - criteria |= Q('match', **{attr: val}) - results = results.query(criteria) + criteria |= Q('term', **{attr: val}) + results = results.filter(criteria) else: - results = results.query('match', **{attr: value}) - + results = results.filter('term', **{attr: value}) + + sort_attrs = self._get_sort_attribute() + if self.is_concept_document() and (not sort_attrs or '_score' in get(sort_attrs, '0', {})): + search_str = self.get_search_string(lower=False) + results = results.extra( + rescore={ + "window_size": 400, + "query": { + "score_mode": "total", + "query_weight": 1.0, + "rescore_query_weight": 800.0, + "rescore_query": { + "dis_max": { + "tie_breaker": 0.0, + "queries": [ + { + "constant_score": { + "filter": { "term": { "_name": { "value": search_str, "case_insensitive": True } } }, + "boost": 10.0 + } + }, + { + "constant_score": { + "filter": { "term": { "_synonyms": { "value": search_str, "case_insensitive": True } } }, + "boost": 8.0 + } + } + ] + } + } + } + } + ) if fields and highlight and self.request.query_params.get(INCLUDE_SEARCH_META_PARAM) in get_truthy_values(): results = results.highlight(*self.clean_fields_for_highlight(fields)) results = results.source(excludes=['_synonyms_embeddings', '_embeddings']) - - return results.sort(*self._get_sort_attribute()) if sort else results + return results.sort(*sort_attrs) if sort else results def get_mandatory_words_criteria(self): criterion = None @@ -970,8 +998,6 @@ def is_repo_version_children_request_without_any_search(self): def should_perform_es_search(self): if self.is_repo_version_children_request() and self.request.query_params.get('onlyHierarchyRoot') not in TRUTHY: return True - if self.is_concept_document() and get(self, 'parent_resource.concept_filter_default'): - return True sort_field, _ = self.get_sort_and_desc() return ( self.is_only_searchable or diff --git a/core/concepts/documents.py b/core/concepts/documents.py index 9d80c7802..8ebd379c0 100644 --- a/core/concepts/documents.py +++ b/core/concepts/documents.py @@ -1,6 +1,5 @@ from django_elasticsearch_dsl import Document, fields from django_elasticsearch_dsl.registries import registry -from elasticsearch_dsl import DenseVector from pydash import compact, get from core.common.utils import jsonify_safe, flatten_dict, get_embeddings, drop_version @@ -90,11 +89,14 @@ def get_match_phrase_attrs(): def get_exact_match_attrs(): return { 'id': { - 'boost': 50 + 'boost': 15 }, 'name': { 'boost': 15 }, + 'synonyms': { + 'boost': 10 + }, 'external_id': { 'boost': 6 }, @@ -110,10 +112,10 @@ def get_exact_match_attrs(): def get_wildcard_search_attrs(): return { 'id': { - 'boost': 35 + 'boost': 5 }, 'name': { - 'boost': 23 + 'boost': 3 }, 'synonyms': { 'boost': 0.3, diff --git a/core/concepts/migrations/0078_auto_20250909_1351.py b/core/concepts/migrations/0078_auto_20250909_1351.py new file mode 100644 index 000000000..3a6214e38 --- /dev/null +++ b/core/concepts/migrations/0078_auto_20250909_1351.py @@ -0,0 +1,15 @@ +# Generated by Django 4.2.16 on 2025-09-09 13:51 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('concepts', '0077_auto_20250226_0548'), + ] + + operations = [ + migrations.RunSQL('create index if not exists concepts_retired on concepts (id) where retired;'), + migrations.RunSQL('create index if not exists concepts_parent_versioned_retired on concepts (parent_id) where id = versioned_object_id and retired;') + ] diff --git a/core/concepts/models.py b/core/concepts/models.py index 302f441e0..5d9764adb 100644 --- a/core/concepts/models.py +++ b/core/concepts/models.py @@ -4,7 +4,7 @@ from django.core.validators import RegexValidator from django.db import models, IntegrityError from django.db.models import F, Q -from pydash import get, compact +from pydash import get, compact, has from core.common.checksums import ChecksumModel from core.common.constants import ISO_639_1, LATEST, HEAD, ALL @@ -648,14 +648,21 @@ def get_source_version_before_creation(self): def properties(self): parent = self.get_parent_source_version() - return self.__get_properties_from_extras_and_definitions(get(parent, 'properties') or []) + return self.__get_properties_from_extras_and_definitions( + get(parent, 'properties') or [], + summary=get(parent, 'concept_summary_properties'), + return_all=True + ) @property def summary_properties(self): parent = self.get_parent_source_version() return self.__get_properties_from_extras_and_definitions( - get(parent, 'properties') or [], summary=get(parent, 'concept_summary_properties') or []) + get(parent, 'properties') or [], + summary=get(parent, 'concept_summary_properties') or [], + return_all=False + ) @property def filters(self): @@ -676,34 +683,41 @@ def filters_ordered(self): return definitions def get_parent_source_version(self): - if self.is_versioned_object or self.is_latest_version: + if self.is_versioned_object: return self.parent - return self.latest_source_version or self.get_source_version_before_creation() + return self.latest_source_version or self.get_source_version_before_creation() or self.parent - def __get_properties_from_extras_and_definitions(self, definitions, summary=False): + def __get_properties_from_extras_and_definitions(self, definitions, summary=False, return_all=True): extras = self.extras or {} result = [] summary_codes = summary or [] + NOT_EXISTING_VALUE = '____FAlse____' def resolve_value(prop): code = prop["code"] if code not in extras and code.lower() in {"concept_class", "class", "conceptclass", "datatype"}: return self.datatype if code.lower() == "datatype" else self.concept_class - return get(extras, code) + return get(extras, code) if has(extras, code) else NOT_EXISTING_VALUE def build_property(prop): if not prop: - return None + return False + value = resolve_value(prop) + if value == NOT_EXISTING_VALUE: + return False value_key = f"value{(prop.get('type') or '').title()}" - return {"code": prop["code"], value_key: resolve_value(prop)} + return {"code": prop["code"], value_key: value} - if summary is False: - for _prop in definitions: - if built := build_property(_prop): - result.append(built) - else: - for prop_code in summary_codes: + for prop_code in summary_codes: + _prop = next((definition for definition in definitions if definition['code'] == prop_code), None) + if built := build_property(_prop): + result.append(built) + + if return_all: + rest = sorted([ + _prop['code'] for _prop in definitions if _prop.get('code') and _prop.get('code') not in summary_codes]) + for prop_code in rest: _prop = next((definition for definition in definitions if definition['code'] == prop_code), None) if built := build_property(_prop): result.append(built) diff --git a/core/concepts/search.py b/core/concepts/search.py index f6c9ff67a..1d8c1d939 100644 --- a/core/concepts/search.py +++ b/core/concepts/search.py @@ -116,7 +116,7 @@ def get_exact_and_contains_criteria(field, value, boost=0, add_boost=True): @classmethod def search( # pylint: disable=too-many-locals,too-many-arguments,too-many-branches,too-many-statements cls, data, repo_url, repo_params=None, include_retired=False, - is_semantic=False, num_candidates=5000, k_nearest=50, map_config=None, additional_filter_criterion=None + is_semantic=False, num_candidates=2000, k_nearest=50, map_config=None, additional_filter_criterion=None ): from core.concepts.documents import ConceptDocument map_config = map_config or [] @@ -135,6 +135,11 @@ def search( # pylint: disable=too-many-locals,too-many-arguments,too-many-branc for val in values: val = val or "" priority_criteria.append(CustomESSearch.get_or_match_criteria(field, val, boost)) + for field, value in data.items(): + if field.startswith('properties__'): + property_code = field.split('properties__', 1)[-1] + priority_criteria.append( + Q('term', **{f"properties.{property_code}.keyword": value.strip('\"').strip('\'')})) knn_queries = [] name = None @@ -201,7 +206,7 @@ def get_knn_query(_field, _value, _boost): if is_semantic: if name: search = search.extra(rescore={ - "window_size": 1000, + "window_size": 250, "query": { "score_mode": "total", "query_weight": 1.0, @@ -249,7 +254,9 @@ def get_knn_query(_field, _value, _boost): } }) - highlight = [field for field in flatten([*cls.fuzzy_fields, *fields]) if not is_number(field)] + highlight = [ + 'name', 'synonyms' + ] if is_semantic else [field for field in flatten([*cls.fuzzy_fields, *fields]) if not is_number(field)] search = search.highlight(*highlight) search = search.sort({'_score': {'order': 'desc'}}) return search diff --git a/core/concepts/tests/tests.py b/core/concepts/tests/tests.py index c548aa731..56d1cc06d 100644 --- a/core/concepts/tests/tests.py +++ b/core/concepts/tests/tests.py @@ -1418,6 +1418,14 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements } source.save() + self.assertEqual( + source.filters_ordered, + [ + {'code': 'concept_class', 'operator': ['='], 'value': 'blah'}, + {'code': 'datatype', 'operator': ['='], 'value': 'blah'}, + ] + ) + concept2 = ConceptFactory(parent=source, concept_class='Diagnosis', datatype='N/A') concept3 = ConceptFactory( parent=source, concept_class='Diagnosis', datatype='N/A', extras={'foo': 'bar', 'units': 'parts/microliter'} @@ -1435,9 +1443,8 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements self.assertEqual( concept.properties, [ - {'code': 'concept_class', 'valueCode': 'Diagnosis'}, {'code': 'datatype', 'valueCode': 'N/A'}, - {'code': 'units', 'valueString': None} + {'code': 'concept_class', 'valueCode': 'Diagnosis'} ] ) self.assertEqual( @@ -1462,8 +1469,8 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements self.assertEqual( concept.properties, [ - {'code': 'concept_class', 'valueCode': 'Diagnosis'}, {'code': 'datatype', 'valueCode': 'N/A'}, + {'code': 'concept_class', 'valueCode': 'Diagnosis'}, {'code': 'units', 'valueString': 'parts/microliter'} ] ) @@ -1490,6 +1497,14 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements } source.save() + self.assertEqual( + source.filters_ordered, + [ + {'code': 'datatype', 'operator': ['='], 'value': 'blah'}, + {'code': 'concept_class', 'operator': ['='], 'value': 'blah'}, + ] + ) + for concept in concept2s: concept.refresh_from_db() self.assertEqual(concept.extras, {}) @@ -1498,7 +1513,6 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements [ {'code': 'concept_class', 'valueCode': 'Diagnosis'}, {'code': 'datatype', 'valueCode': 'N/A'}, - {'code': 'units', 'valueString': None} ] ) self.assertEqual( @@ -1543,8 +1557,32 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements ] ) - source.meta = {'display': {'concept_summary_properties': ['concept_class', 'datatype', 'foobar']}} + source.meta = { + 'display': { + 'concept_summary_properties': ['concept_class', 'datatype', 'foobar'], + 'concept_filter_order': ['datatype', 'concept_class', 'foobar', 'barbar', 'bar1'], + }, + } + source.filters = [ + {'code': 'concept_class', 'operator': ['='], 'value': 'blah'}, + {'code': 'datatype', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar2', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar0', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar1', 'operator': ['='], 'value': 'blah'}, + {'code': 'barbar', 'operator': ['='], 'value': 'blah'}, + ] source.save() + self.assertEqual( + source.filters_ordered, + [ + {'code': 'datatype', 'operator': ['='], 'value': 'blah'}, + {'code': 'concept_class', 'operator': ['='], 'value': 'blah'}, + {'code': 'barbar', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar1', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar0', 'operator': ['='], 'value': 'blah'}, + {'code': 'bar2', 'operator': ['='], 'value': 'blah'}, + ] + ) for concept in concept2s: concept.refresh_from_db() @@ -1553,8 +1591,7 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements concept.properties, [ {'code': 'concept_class', 'valueCode': 'Diagnosis'}, - {'code': 'datatype', 'valueCode': 'N/A'}, - {'code': 'units', 'valueString': None} + {'code': 'datatype', 'valueCode': 'N/A'} ] ) self.assertEqual( @@ -1595,8 +1632,7 @@ def test_properties_and_filters(self): # pylint: disable=too-many-statements concept.properties, [ {'code': 'concept_class', 'valueCode': 'Diagnosis'}, - {'code': 'datatype', 'valueCode': 'N/A'}, - {'code': 'units', 'valueString': None} + {'code': 'datatype', 'valueCode': 'N/A'} ] ) self.assertEqual( diff --git a/core/concepts/views.py b/core/concepts/views.py index 6a81aa315..f2962431a 100644 --- a/core/concepts/views.py +++ b/core/concepts/views.py @@ -805,7 +805,7 @@ def filter_queryset(self, _=None): # pylint:disable=too-many-locals,too-many-st map_config = self.request.data.get('map_config', []) filters = self.request.data.get('filter', {}) include_retired = self.request.query_params.get(INCLUDE_RETIRED_PARAM) in get_truthy_values() - num_candidates = min(to_int(self.request.query_params.get('numCandidates', 0), 5000), 5000) + num_candidates = min(to_int(self.request.query_params.get('numCandidates', 0), 2000), 2000) k_nearest = min(to_int(self.request.query_params.get('kNearest', 0), 50), 50) offset = max(to_int(self.request.GET.get('offset'), 0), 0) limit = max(to_int(self.request.GET.get('limit'), 0), 0) or self.default_limit @@ -820,15 +820,21 @@ def filter_queryset(self, _=None): # pylint:disable=too-many-locals,too-many-st faceted_criterion = self.get_faceted_criterion(False, filters, minimum_should_match=1) if filters else None results = [] + import time for row in rows: + start_time = time.time() search = ConceptFuzzySearch.search( row, target_repo_url, repo_params, include_retired, is_semantic, num_candidates, k_nearest, map_config, faceted_criterion ) - search = search.params(min_score=score_threshold if best_match else 0) + print("Search Query", time.time() - start_time) + start_time = time.time() + search = search.params(track_total_hits=False, request_cache=True) es_search = CustomESSearch(search[start:end], ConceptDocument) - es_search.to_queryset(False, True) + es_search.to_queryset(False, True, False) + print("Search to Queryset", time.time() - start_time) result = {'row': row, 'results': [], 'map_config': map_config, 'filter': filters} + start_time = time.time() for concept in es_search.queryset: concept._highlight = es_search.highlights.get(concept.id, {}) # pylint:disable=protected-access score_info = es_search.scores.get(concept.id, {}) @@ -836,28 +842,27 @@ def filter_queryset(self, _=None): # pylint:disable=too-many-locals,too-many-st normalized_score = get(score_info, 'normalized') or None concept._score = score # pylint:disable=protected-access concept._normalized_score = normalized_score # pylint:disable=protected-access - concept._match_type = 'low' # pylint:disable=protected-access - # Use normalized score for thresholding if available - score_to_check = normalized_score if normalized_score is not None else score - if score_to_check is not None: - if is_semantic and score_to_check > 0.9: + if limit > 1: + concept._match_type = 'low' # pylint:disable=protected-access + score_to_check = normalized_score if normalized_score is not None else score + if concept._highlight.get('name', None) or is_semantic and score_to_check > score_threshold: # pylint:disable=protected-access concept._match_type = 'very_high' # pylint:disable=protected-access - elif score_to_check > 0.6: + elif concept._highlight.get('synonyms', None): # pylint:disable=protected-access concept._match_type = 'high' # pylint:disable=protected-access - # Optionally, keep existing highlight logic - if concept._highlight.get('name', None): # pylint:disable=protected-access - concept._match_type = 'very_high' # pylint:disable=protected-access - if is_semantic and score_to_check > self.score_threshold_semantic_very_high: # pylint:disable=protected-access,line-too-long - concept._match_type = 'very_high' # pylint:disable=protected-access - if not best_match or concept._match_type == 'very_high': # pylint:disable=protected-access + else: + concept._match_type = 'very_high' # pylint:disable=protected-access + if not best_match or concept._match_type in ['high', 'very_high']: # pylint:disable=protected-access serializer = ConceptDetailSerializer if self.is_verbose() else ConceptMinimalSerializer data = serializer(concept, context={'request': self.request}).data data['search_meta']['search_normalized_score'] = normalized_score * 100 result['results'].append(data) + print("Queryset to Serializer", time.time() - start_time) + start_time = time.time() if 'results' in result: result['results'] = sorted( result['results'], key=lambda res: get(res, 'search_meta.search_normalized_score'), reverse=True) results.append(result) + print("Sorting", time.time() - start_time) return results diff --git a/core/events/migrations/0010_auto_20250910_0142.py b/core/events/migrations/0010_auto_20250910_0142.py new file mode 100644 index 000000000..d076e7f8a --- /dev/null +++ b/core/events/migrations/0010_auto_20250910_0142.py @@ -0,0 +1,15 @@ +# Generated by Django 4.2.16 on 2025-09-10 01:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('events', '0009_alter_event_referenced_object_url'), + ] + + operations = [ + migrations.RunSQL('CREATE INDEX if not exists idx_events_object_url ON events_event (object_url);'), + migrations.RunSQL('CREATE INDEX if not exists idx_events_referenced_object_url ON events_event (referenced_object_url);') + ] diff --git a/core/events/models.py b/core/events/models.py index 29d2e704f..2719b2ac9 100644 --- a/core/events/models.py +++ b/core/events/models.py @@ -1,5 +1,5 @@ from django.db import models -from pydash import has +from pydash import has, compact class Event(models.Model): @@ -47,18 +47,14 @@ def get_events_for_following(cls, following_queryset, private=False, **event_kwa @classmethod def get_user_organization_events(cls, user, private=False): - criterion = None - for org in user.organizations.filter(): - criteria = Event.object_criteria(org.uri) - repo_events_criteria = org.get_repo_events_criteria(private) - if repo_events_criteria is not None: - criteria |= repo_events_criteria - if criterion is None: - criterion = criteria - else: - criterion |= criteria - - queryset = Event.objects.none() if criterion is None else Event.objects.filter(criterion) + org_uris = list(user.organizations.values_list('uri', flat=True)) + source_uris = compact(user.organizations.values_list('source__uri', flat=True)) + collection_uris = compact(user.organizations.values_list('collection__uri', flat=True)) + + queryset = Event.objects.filter( + models.Q(object_url__in=org_uris) | + models.Q(referenced_object_url__in=[*org_uris, *source_uris, *collection_uris]) + ) return queryset if private else queryset.filter(public=True) diff --git a/core/events/views.py b/core/events/views.py index 9cd01b296..e83514a06 100644 --- a/core/events/views.py +++ b/core/events/views.py @@ -17,12 +17,17 @@ class EventsView(BaseAPIView, ListWithHeadersMixin): permission_classes = (AllowAny,) default_qs_sort_attr = '-created_at' serializer_class = EventSerializer + owner = None def get_owner(self): - owner = self.get_owner_from_kwargs() - if not owner: + if self.owner: + return self.owner + + self.owner = self.get_owner_from_kwargs() + if not self.owner: raise Http404() - return owner + + return self.owner def get_queryset(self): owner = self.get_owner() diff --git a/core/fixtures/auth_groups.yaml b/core/fixtures/auth_groups.yaml index 7f565e0e9..ff7f61b60 100644 --- a/core/fixtures/auth_groups.yaml +++ b/core/fixtures/auth_groups.yaml @@ -14,3 +14,7 @@ pk: 4 fields: name: operations_panel +- model: "auth.group" + pk: 5 + fields: + name: mapper_ai_assistant diff --git a/core/importers/tests.py b/core/importers/tests.py index 3e296b40b..4b9caf064 100644 --- a/core/importers/tests.py +++ b/core/importers/tests.py @@ -719,7 +719,7 @@ def test_reference_import(self, batch_index_resources_mock): self.assertEqual(collection.expansions.count(), 1) self.assertEqual(collection.expansion.concepts.count(), 4) self.assertEqual(collection.expansion.mappings.count(), 0) - self.assertEqual(collection.references.count(), 4) + self.assertEqual(collection.references.count(), 5) # duplicate run importer = BulkImportInline( @@ -742,7 +742,7 @@ def test_reference_import(self, batch_index_resources_mock): self.assertEqual(collection.expansions.count(), 1) self.assertEqual(collection.expansion.concepts.count(), 4) self.assertEqual(collection.expansion.mappings.count(), 0) - self.assertEqual(collection.references.count(), 4) + self.assertEqual(collection.references.count(), 10) batch_index_resources_mock.apply_async.assert_not_called() @patch('core.collections.models.batch_index_resources', Mock()) diff --git a/core/integration_tests/tests_collections.py b/core/integration_tests/tests_collections.py index e2f1b513f..77021cf51 100644 --- a/core/integration_tests/tests_collections.py +++ b/core/integration_tests/tests_collections.py @@ -656,7 +656,7 @@ def test_delete_204_specific_expression(self): ) self.assertEqual(response.status_code, 200) self.collection.refresh_from_db() - self.assertEqual(self.collection.references.count(), 2) + self.assertEqual(self.collection.references.count(), 1) self.assertEqual(self.collection.expansion.concepts.count(), 1) self.assertEqual(self.collection.expansion.mappings.count(), 1) @@ -670,9 +670,9 @@ def test_delete_204_specific_expression(self): ) self.assertEqual(response.status_code, 204) self.collection.refresh_from_db() - self.assertEqual(self.collection.references.count(), 1) + self.assertEqual(self.collection.references.count(), 0) self.assertEqual(self.collection.expansion.concepts.count(), 0) - self.assertEqual(self.collection.expansion.mappings.count(), 1) + self.assertEqual(self.collection.expansion.mappings.count(), 0) @patch('core.collections.views.add_references') def test_put_202_all(self, add_references_mock): @@ -728,16 +728,10 @@ def test_put_200_specific_expression(self): # pylint: disable=too-many-statemen response.data, [ { - 'added': False, + 'added': True, 'expression': self.concept.uri, - 'message': { - self.concept.uri: { - 'errors': [{ - 'description': 'Concept or Mapping reference name must be unique in a collection.', - 'conflicting_references': [self.reference.uri] - }] - } - } + 'message': f'The concept {self.concept.mnemonic} is successfully added to ' + f'collection {self.collection.name}' } ] ) @@ -756,7 +750,7 @@ def test_put_200_specific_expression(self): # pylint: disable=too-many-statemen self.assertEqual(response.status_code, 200) self.collection.refresh_from_db() - self.assertEqual(self.collection.references.count(), 2) + self.assertEqual(self.collection.references.count(), 3) self.assertEqual(self.collection.expansion.concepts.count(), 2) self.assertEqual(self.collection.active_concepts, 2) self.assertEqual(self.collection.active_mappings, 0) @@ -788,7 +782,7 @@ def test_put_200_specific_expression(self): # pylint: disable=too-many-statemen self.assertEqual(response.status_code, 200) self.collection.refresh_from_db() - self.assertEqual(self.collection.references.count(), 3) + self.assertEqual(self.collection.references.count(), 4) self.assertEqual(self.collection.expansion.concepts.count(), 2) self.assertEqual(self.collection.expansion.mappings.count(), 1) self.assertEqual(self.collection.active_concepts, 2) @@ -891,7 +885,7 @@ def test_put_expression_with_cascade_to_concepts(self): concept3 = ConceptFactory(parent=source2) concept4 = ConceptFactory(parent=source2) - mapping1 = MappingFactory( + MappingFactory( mnemonic='m1-c1-c2-s1', from_concept=concept1.get_latest_version(), to_concept=concept2.get_latest_version(), parent=source1 ) @@ -903,7 +897,7 @@ def test_put_expression_with_cascade_to_concepts(self): mnemonic='m3-c1-c3-s2', from_concept=concept1.get_latest_version(), to_concept=concept3.get_latest_version(), parent=source2 ) - mapping4 = MappingFactory( + MappingFactory( mnemonic='m4-c4-c3-s2', from_concept=concept4.get_latest_version(), to_concept=concept3.get_latest_version(), parent=source2 ) @@ -924,14 +918,11 @@ def test_put_expression_with_cascade_to_concepts(self): ) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 3) + self.assertEqual(len(response.data), 1) self.assertTrue(all(data['added'] for data in response.data)) self.assertEqual( sorted([data['expression'] for data in response.data]), - sorted([ - concept1.get_latest_version().uri, mapping1.uri, - mapping1.to_concept.versioned_object.uri - ]) + sorted([concept1.get_latest_version().uri]) ) reference = self.collection.references.last() self.assertEqual(reference.cascade, 'sourcetoconcepts') @@ -952,14 +943,11 @@ def test_put_expression_with_cascade_to_concepts(self): ) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 3) + self.assertEqual(len(response.data), 1) self.assertTrue(all(data['added'] for data in response.data)) self.assertEqual( sorted([data['expression'] for data in response.data]), - sorted([ - concept4.get_latest_version().uri, mapping4.uri, - mapping4.to_concept.versioned_object.uri - ]) + sorted([concept4.get_latest_version().uri]) ) random_concept = ConceptFactory() @@ -1600,7 +1588,7 @@ def test_post_200_specific_expression(self): # pylint: disable=too-many-stateme format='json' ) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 2) + self.assertEqual(len(response.data), 1) self.assertEqual( sorted(response.data, key=lambda ref: ref['reference']), [ @@ -1611,14 +1599,6 @@ def test_post_200_specific_expression(self): # pylint: disable=too-many-stateme 'concepts_count': 1, 'mappings_count': 1, 'exclude': False - }, - { - 'reference': mapping.uri, - 'concepts': [], - 'mappings': [MappingListSerializer(mapping).data], - 'concepts_count': 0, - 'mappings_count': 1, - 'exclude': False } ] ) @@ -1725,7 +1705,7 @@ def test_post_expression_with_cascade_to_concepts(self): ) self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 3) + self.assertEqual(len(response.data), 1) self.assertEqual( sorted(response.data, key=lambda ref: ref['reference']), [ @@ -1736,22 +1716,6 @@ def test_post_expression_with_cascade_to_concepts(self): 'mappings': [MappingListSerializer(mapping1).data], 'mappings_count': 1, 'exclude': False - }, - { - 'reference': concept2.uri, - 'concepts': ConceptListSerializer([concept2], many=True).data, - 'concepts_count': 1, - 'mappings': [], - 'mappings_count': 0, - 'exclude': False - }, - { - 'reference': mapping1.uri, - 'concepts': [], - 'concepts_count': 0, - 'mappings': [MappingListSerializer(mapping1).data], - 'mappings_count': 1, - 'exclude': False } ] ) @@ -1767,18 +1731,10 @@ def test_post_expression_with_cascade_to_concepts(self): format='json' ) - self.assertEqual(len(response.data), 3) + self.assertEqual(len(response.data), 1) self.assertEqual( sorted(response.data, key=lambda ref: ref['reference']), [ - { - 'reference': concept3.uri, - 'concepts': ConceptListSerializer([concept3], many=True).data, - 'concepts_count': 1, - 'mappings': [], - 'mappings_count': 0, - 'exclude': False - }, { 'reference': concept4.get_latest_version().uri, 'concepts': ConceptListSerializer([concept3, concept4.get_latest_version()], many=True).data, @@ -1786,14 +1742,6 @@ def test_post_expression_with_cascade_to_concepts(self): 'mappings': [MappingListSerializer(mapping4).data], 'mappings_count': 1, 'exclude': False - }, - { - 'reference': mapping4.uri, - 'concepts': [], - 'concepts_count': 0, - 'mappings': [MappingListSerializer(mapping4).data], - 'mappings_count': 1, - 'exclude': False } ] ) diff --git a/core/integration_tests/tests_orgs.py b/core/integration_tests/tests_orgs.py index f1200cf0a..ef49d662d 100644 --- a/core/integration_tests/tests_orgs.py +++ b/core/integration_tests/tests_orgs.py @@ -1,3 +1,6 @@ +import unittest + +from django.conf import settings from mock import patch from mock.mock import Mock, ANY from rest_framework.exceptions import ErrorDetail @@ -596,16 +599,7 @@ def test_get(self): response = self.client.get('/users/batman/orgs/sources/') - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 2) - self.assertEqual( - [data['short_code'] for data in response.data], - ['corporate', 'city'] - ) - self.assertEqual( - [data['owner_url'] for data in response.data], - ['/orgs/wayne-enterprise/', '/orgs/gotham/'] - ) + self.assertEqual(response.status_code, 401) response = self.client.get( '/user/orgs/sources/', @@ -619,6 +613,20 @@ def test_get(self): ['corporate', 'city'] ) + @unittest.skipIf(settings.ENV == 'ci', "Skipping due to ES tests failing on CI") + def test_get_with_search(self): + user = UserProfileFactory(username='batman') + token = user.get_token() + org1 = OrganizationFactory(mnemonic='gotham') + org2 = OrganizationFactory(mnemonic='wayne-enterprise') + org1.members.add(user) + org2.members.add(user) + source1 = OrganizationSourceFactory(mnemonic='city', organization=org1) + source2 = OrganizationSourceFactory(mnemonic='corporate', organization=org2) + source3 = UserSourceFactory(mnemonic='bat-cave', user=user) + + SourceDocument().update([source1, source2, source3]) + response = self.client.get( '/user/orgs/sources/?q=city', HTTP_AUTHORIZATION=f'Token {token}' @@ -656,16 +664,7 @@ def test_get(self): response = self.client.get('/users/batman/orgs/collections/') - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 2) - self.assertEqual( - [data['short_code'] for data in response.data], - ['corporate', 'city'] - ) - self.assertEqual( - [data['owner_url'] for data in response.data], - ['/orgs/wayne-enterprise/', '/orgs/gotham/'] - ) + self.assertEqual(response.status_code, 401) response = self.client.get( '/user/orgs/collections/', @@ -679,6 +678,20 @@ def test_get(self): ['corporate', 'city'] ) + @unittest.skipIf(settings.ENV == 'ci', "Skipping due to ES tests failing on CI") + def test_get_with_search(self): + user = UserProfileFactory(username='batman') + token = user.get_token() + org1 = OrganizationFactory(mnemonic='gotham') + org2 = OrganizationFactory(mnemonic='wayne-enterprise') + org1.members.add(user) + org2.members.add(user) + coll1 = OrganizationCollectionFactory(mnemonic='city', organization=org1) + coll2 = OrganizationCollectionFactory(mnemonic='corporate', organization=org2) + coll3 = UserCollectionFactory(mnemonic='bat-cave', user=user) + + CollectionDocument().update([coll1, coll2, coll3]) + response = self.client.get( '/user/orgs/collections/?q=city', HTTP_AUTHORIZATION=f'Token {token}' diff --git a/core/map_projects/migrations/0017_mapproject_filters.py b/core/map_projects/migrations/0017_mapproject_filters.py new file mode 100644 index 000000000..9157bb71a --- /dev/null +++ b/core/map_projects/migrations/0017_mapproject_filters.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2025-09-22 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('map_projects', '0016_mapproject_batch_size'), + ] + + operations = [ + migrations.AddField( + model_name='mapproject', + name='filters', + field=models.JSONField(blank=True, default=dict, null=True), + ), + ] diff --git a/core/map_projects/models.py b/core/map_projects/models.py index 3be06a05f..0399ff4c9 100644 --- a/core/map_projects/models.py +++ b/core/map_projects/models.py @@ -34,6 +34,7 @@ class MapProject(BaseModel): include_retired = models.BooleanField(default=False) logs = models.JSONField(default=dict, null=True, blank=True) score_configuration = models.JSONField(default=default_score_configuration, null=True, blank=True) + filters = models.JSONField(default=dict, null=True, blank=True) # Custom API match_api_url = models.TextField(null=True, blank=True) @@ -165,6 +166,7 @@ def format_request_data(cls, data, parent_resource=None): cls.format_json(new_data, 'matches') cls.format_json(new_data, 'columns') cls.format_json(new_data, 'score_configuration') + cls.format_json(new_data, 'filters') if parent_resource: new_data[parent_resource.resource_type.lower() + '_id'] = parent_resource.id @@ -187,6 +189,8 @@ def soft_delete(self): self.delete() def clean(self): + if not self.filters: + self.filters = {} if not self.batch_size: self.batch_size = self.BATCH_SIZE if not self.include_retired: @@ -196,3 +200,23 @@ def clean(self): self.matches = json.loads(self.matches) except (json.JSONDecodeError, TypeError): pass + + @property + def target_repo(self): + if not self.target_repo_url: + return None + + from core.sources.models import Source + repo, _ = Source.resolve_reference_expression(self.target_repo_url) + return repo if repo and repo.id else None + + @property + def fields_mapped(self): + return [ + col.get('label') for col in self.visible_columns if ( + col['label'].lower() in [ + 'id', 'description', 'mapping: list', 'mapping: code', + 'concept_class', 'class', 'datatype', 'name', 'synonyms' + ] or col['label'].lower().startswith('property:') + ) + ] if self.columns else [] diff --git a/core/map_projects/serializers.py b/core/map_projects/serializers.py index 037ba1128..c611dab84 100644 --- a/core/map_projects/serializers.py +++ b/core/map_projects/serializers.py @@ -22,7 +22,7 @@ class Meta: 'created_by', 'updated_by', 'created_at', 'updated_at', 'url', 'is_active', 'public_access', 'file', 'user_id', 'organization_id', 'description', 'target_repo_url', 'matching_algorithm', 'include_retired', 'score_configuration', - 'match_api_url', 'match_api_token', 'batch_size' + 'match_api_url', 'match_api_token', 'batch_size', 'filters' ] def prepare_object(self, validated_data, instance=None, file=None): @@ -36,7 +36,7 @@ def prepare_object(self, validated_data, instance=None, file=None): instance.columns = columns for attr in [ 'name', 'description', 'extras', 'target_repo_url', 'matching_algorithm', 'include_retired', - 'score_configuration', 'match_api_url', 'match_api_token', 'batch_size' + 'score_configuration', 'match_api_url', 'match_api_token', 'batch_size', 'filters' ]: setattr(instance, attr, validated_data.get(attr, get(instance, attr))) if not instance.id: @@ -90,7 +90,7 @@ class Meta: 'created_by', 'updated_by', 'created_at', 'updated_at', 'url', 'is_active', 'owner', 'owner_type', 'owner_url', 'public_access', 'target_repo_url', 'matching_algorithm', 'summary', 'logs', 'include_retired', - 'score_configuration', 'match_api_url', 'match_api_token', 'batch_size' + 'score_configuration', 'match_api_url', 'match_api_token', 'batch_size', 'filters' ] def __init__(self, *args, **kwargs): diff --git a/core/map_projects/urls.py b/core/map_projects/urls.py index 5f68df07c..5876c77d8 100644 --- a/core/map_projects/urls.py +++ b/core/map_projects/urls.py @@ -5,6 +5,7 @@ urlpatterns = [ path('', views.MapProjectListView.as_view(), name='map-project-list'), path('/', views.MapProjectView.as_view(), name='map-project'), + path('/recommend-beta/', views.MapProjectRecommendView.as_view(), name='map-project-recommend'), path('/summary/', views.MapProjectSummaryView.as_view(), name='map-project-summary'), path('/logs/', views.MapProjectLogsView.as_view(), name='map-project-logs'), ] diff --git a/core/map_projects/views.py b/core/map_projects/views.py index 14b1f90b2..99347c79a 100644 --- a/core/map_projects/views.py +++ b/core/map_projects/views.py @@ -1,10 +1,11 @@ - +from django.conf import settings from rest_framework import status from rest_framework.generics import RetrieveUpdateDestroyAPIView, RetrieveAPIView, CreateAPIView from rest_framework.response import Response from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin -from core.common.permissions import CanViewConceptDictionary, HasOwnership +from core.common.permissions import HasOwnership, IsInAuthGroup, CanEditConceptDictionary +from core.common.utils import get_truthy_values from core.common.views import BaseAPIView from core.map_projects.models import MapProject from core.map_projects.serializers import MapProjectSerializer, MapProjectCreateUpdateSerializer, \ @@ -14,7 +15,7 @@ class MapProjectBaseView(BaseAPIView): is_searchable = False queryset = MapProject.objects.filter(is_active=True) - permission_classes = (CanViewConceptDictionary,) + permission_classes = (CanEditConceptDictionary,) serializer_class = MapProjectSerializer @@ -61,6 +62,46 @@ def get_serializer_class(self): return self.serializer_class +class MapProjectRecommendView(MapProjectBaseView): # pragma: no cover + serializer_class = MapProjectDetailSerializer + lookup_url_kwarg = 'project' + lookup_field = 'project' + pk_field = 'id' + permission_classes = (IsInAuthGroup,) + swagger_schema = None + + def post(self, request, *args, **kwargs): # pylint: disable=unused-argument + params = self.request.query_params + map_project = self.get_object() + candidates = request.data.get('candidates') or [] + row = request.data.get('row') or {} + target_repo_url = request.data.get('target_repo_url') or map_project.target_repo_url + + if not candidates or not isinstance(candidates, list) or not row or not isinstance(row, dict): + return Response( + {'detail': 'candidates (list) and row (dict) are required.'}, status=status.HTTP_400_BAD_REQUEST + ) + if not target_repo_url: + return Response( + {'detail': 'target_repo_url is required either in the request body or the map project.'}, + status=status.HTTP_400_BAD_REQUEST + ) + + from core.services.litellm import LiteLLMService + if not settings.ENV or settings.ENV in ['ci', 'development', 'test']: + return Response(LiteLLMService.mock_response) + + try: + litellm = LiteLLMService() + map_project.target_repo_url = target_repo_url + response = litellm.recommend( + map_project, row, candidates, params.get('conceptFilterDefault') in get_truthy_values() + ) + return Response(litellm.to_dict(response), status=status.HTTP_200_OK) + except Exception as ex: + return Response({'detail': str(ex)}, status=status.HTTP_400_BAD_REQUEST) + + class MapProjectSummaryView(MapProjectBaseView, RetrieveAPIView): serializer_class = MapProjectSummarySerializer lookup_url_kwarg = 'project' diff --git a/core/orgs/documents.py b/core/orgs/documents.py index dbb0a398c..d129cb2e9 100644 --- a/core/orgs/documents.py +++ b/core/orgs/documents.py @@ -22,13 +22,13 @@ class Index: _mnemonic = fields.KeywordField(attr='mnemonic', normalizer='lowercase') extras = fields.ObjectField(dynamic=True) user = fields.ListField(fields.TextField()) + created_on = fields.DateField(attr='created_at') + location = fields.KeywordField(attr='location') + company = fields.KeywordField(attr='company') class Django: model = Organization - fields = [ - 'company', - 'location', - ] + fields = [] @staticmethod def get_match_phrase_attrs(): diff --git a/core/orgs/models.py b/core/orgs/models.py index 99e25f0d6..7383837b2 100644 --- a/core/orgs/models.py +++ b/core/orgs/models.py @@ -26,8 +26,9 @@ class Meta: '_mnemonic': {'sortable': True, 'filterable': False, 'exact': False}, 'last_update': {'sortable': True, 'default': 'desc', 'filterable': False}, 'updated_by': {'sortable': False, 'filterable': False, 'facet': True}, - 'company': {'sortable': False, 'filterable': True, 'exact': True}, - 'location': {'sortable': False, 'filterable': True, 'exact': True}, + 'company': {'sortable': True, 'filterable': True, 'exact': True}, + 'location': {'sortable': True, 'filterable': True, 'exact': True}, + 'created_on': {'sortable': True, 'filterable': False, 'exact': False}, } name = models.TextField() diff --git a/core/orgs/views.py b/core/orgs/views.py index 55977388c..73d9fc0a7 100644 --- a/core/orgs/views.py +++ b/core/orgs/views.py @@ -4,7 +4,7 @@ from pydash import get from rest_framework import mixins, status, generics from rest_framework.generics import RetrieveAPIView, DestroyAPIView, RetrieveUpdateDestroyAPIView, UpdateAPIView -from rest_framework.permissions import IsAuthenticatedOrReadOnly, AllowAny +from rest_framework.permissions import IsAuthenticatedOrReadOnly, AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView @@ -19,17 +19,16 @@ from core.common.utils import parse_updated_since_param, get_truthy_values from core.common.views import BaseAPIView, BaseLogoView from core.map_projects.views import MapProjectListView -from core.tasks.mixins import TaskMixin from core.orgs.constants import NO_MEMBERS from core.orgs.documents import OrganizationDocument from core.orgs.models import Organization from core.orgs.serializers import OrganizationDetailSerializer, OrganizationListSerializer, \ OrganizationCreateSerializer, OrganizationOverviewSerializer from core.sources.views import SourceListView +from core.tasks.mixins import TaskMixin from core.users.models import UserProfile from core.users.serializers import UserDetailSerializer - TRUTHY = get_truthy_values() @@ -246,6 +245,7 @@ def delete(self, request, **kwargs): # pylint: disable=unused-argument class OrganizationResourceAbstractListView: version = None + permission_classes = (IsAuthenticated,) def get_queryset(self): username = self.kwargs.get('user', None) diff --git a/core/services/litellm.py b/core/services/litellm.py new file mode 100644 index 000000000..9885c6acb --- /dev/null +++ b/core/services/litellm.py @@ -0,0 +1,231 @@ +# pylint: disable=line-too-long +import json + +from django.conf import settings +from litellm import completion +from pydash import get + + +class LiteLLMService: + ANTHROPIC_MODEL = "anthropic/claude-sonnet-4-20250514" + + RECOMMEND_CANDIDATE_SYSTEM_PROMPT = """ + You are an expert medical terminology curator evaluating candidate matches for standardizing local clinical terms to international medical terminologies. + Your role is to assess candidate concepts returned by matching algorithms and provide structured recommendations that prioritize clinical accuracy, semantic precision, and implementation safety. + + ### Core Objectives + - **Clinical Safety**: Ensure matches preserve critical clinical meaning and prevent misinterpretation + - **Semantic Precision**: Select candidates that best capture the intended clinical concept + - **Implementation Viability**: Consider practical constraints like data types, hierarchies, and system compatibility + - **Quality Assurance**: Flag ambiguous or potentially problematic matches for human review + + ### Methodology + 1. Analyze the input term's clinical context, concept class, and intended use + 2. Evaluate each candidate's semantic alignment, specificity, and clinical appropriateness + 3. Consider algorithm confidence scores alongside clinical judgment + 4. Apply project-specific rules and constraints + 5. Provide structured recommendations with clear rationale + + ### Decision Framework + + **RECOMMEND**: Single candidate with high semantic alignment (>85% confidence or clear clinical match) + **CONDITIONAL**: Good candidate(s) exist but with specific limitations or requirements + **REJECT**: No candidates meet minimum quality thresholds + **INSUFFICIENT**: Cannot make confident assessment with available information + + You must respond with a valid JSON object following the specified output template. + """ + + RECOMMEND_CANDIDATE_INPUT_PROMPT = """ + Evaluate the following medical terminology matching task: + + ## Project Context + {project} + + ## Input Row + {row} + + ## Candidate Pool + {candidates} + """ + + RECOMMEND_CANDIDATE_TASK_PROMPT = """ + ## Task + Please evaluate these candidates and provide your recommendation following the structured output template. Focus on: + 1. Semantic alignment between the input term and candidates + 2. Clinical safety and appropriateness + 3. Implementation viability + + Respond with a JSON object following this structure: + { + "recommendation": "RECOMMEND|CONDITIONAL|REJECT|INSUFFICIENT", + "primary_candidate": { + "concept_id": "[Selected concept ID or null]", + "confidence_level": "HIGH|MEDIUM|LOW", + "match_strength": "[Semantic alignment percentage]" + }, + "alternative_candidates": [ + { + "concept_id": "[Alternative concept ID]", + "rank": "[Ranking order]", + "rationale": "[Why this is an alternative]" + } + ], + "conditions_and_caveats": [ + "[Any specific conditions for CONDITIONAL recommendations]" + ], + "rationale": { + "structured": { + "semantic_alignment": "[Assessment of meaning preservation]", + "specificity_level": "[Too broad/appropriate/too narrow]", + "clinical_safety": "[Risk assessment]", + "algorithm_consensus": "[Agreement across algorithms]", + "implementation_complexity": "[Easy/Medium/Complex]", + "data_compatibility": "[Compatible/Requires mapping/Incompatible]" + }, + "narrative": "[2-3 sentence explanation of the recommendation and key factors]" + }, + "quality_flags": [ + "[Any concerns or notable observations]" + ], + "additional_information_needed": [ + "[For INSUFFICIENT recommendations, specify what's needed]" + ] + } + + Important: Your response must be a valid JSON object only, with no additional text or explanation outside the JSON. + """ + + mock_response = { + 'id': 'chatcmpl-4531c7dd-e464-4d8b-a441-ecb2a4ce3940', + 'created': 1757647505, + 'model': 'claude-sonnet-4-20250514', + 'object': 'chat.completion', + 'system_fingerprint': None, + 'choices': [{ + 'finish_reason': 'stop', + 'index': 0, + 'message': { + 'content': { + 'recommendation': 'RECOMMEND', + 'primary_candidate': { + 'concept_id': '1305', + 'confidence_level': 'HIGH', + 'match_strength': '100%' + }, + 'alternative_candidates': [{ + 'concept_id': '856', + 'rank': 2, + 'rationale': 'Generic HIV viral load test without qualitative specification - less precise but related concept' + }], + 'conditions_and_caveats': [], + 'rationale': { + 'structured': { + 'semantic_alignment': 'Perfect match - identical terminology with exact preservation of clinical meaning', + 'specificity_level': 'Appropriate - maintains the qualitative nature distinction from quantitative viral load tests', + 'clinical_safety': 'Excellent - no risk of misinterpretation, preserves critical distinction between qualitative and quantitative testing', + 'algorithm_consensus': 'Strong consensus with normalized score of 100.0 and exact term highlighting', + 'implementation_complexity': 'Easy - direct mapping with no transformation required', + 'data_compatibility': 'Fully compatible - both are Test class concepts with Coded datatype expectations' + }, + 'narrative': 'The primary candidate (1305) provides an exact lexical match with perfect semantic alignment, preserving the critical clinical distinction between qualitative and quantitative HIV viral load testing. This match poses no clinical safety risks and requires no implementation complexity, making it the optimal choice for standardization.' + }, + 'quality_flags': [ + 'Exceptionally high confidence match with 100% normalized search score', + 'Exact term match eliminates ambiguity'], + 'additional_information_needed': [] + }, + 'role': 'assistant', + 'tool_calls': None, + 'function_call': None + } + }], + 'usage': { + 'completion_tokens': 408, + 'prompt_tokens': 3781, + 'total_tokens': 4189, + 'completion_tokens_details': None, + 'prompt_tokens_details': { + 'audio_tokens': None, + 'cached_tokens': 0, + 'text_tokens': None, + 'image_tokens': None + }, + 'cache_creation_input_tokens': 0, + 'cache_read_input_tokens': 0 + } + } + + def __init__(self): + self.anthropic_api_key = settings.ANTHROPIC_API_KEY + + def recommend(self, map_project, row, candidates, include_default_filter=False): # pragma: no cover + prompt = self.get_prompt(map_project, row, candidates, include_default_filter) + response = self.__call_anthropic(prompt) + print("****ANT RESPONSE****") + return response + + @staticmethod + def to_dict(response): # pragma: no cover + data = response.to_dict() + try: + content = get(data, 'choices.0.message.content', '').replace('```json', '').replace('```', '') + parsed_content = json.loads(content) + if parsed_content: + data['choices'][0]['message']['content'] = parsed_content + except (json.JSONDecodeError, TypeError): + pass + return data + + def get_prompt(self, map_project, row, candidates, include_default_filter=False): # pragma: no cover + project_context = self.get_project_context( + map_project, include_default_filter=include_default_filter) + if not project_context: + raise ValueError("Map project must have a valid target repository.") + + system_prompt = self.RECOMMEND_CANDIDATE_SYSTEM_PROMPT.strip() + input_prompt = self.RECOMMEND_CANDIDATE_INPUT_PROMPT.strip().format( + project=json.dumps(project_context, indent=2), + row=json.dumps(row, indent=2), + candidates=json.dumps(candidates, indent=2) + ) + task_prompt = self.RECOMMEND_CANDIDATE_TASK_PROMPT.strip() + + full_prompt = f"{system_prompt}\n\n{input_prompt}\n\n{task_prompt}" + return full_prompt + + @staticmethod + def get_project_context(map_project, include_default_filter=False): # pragma: no cover + target_repo = map_project.target_repo + project_filters = { + **(map_project.filters or {}), + **(target_repo.concept_filter_default if include_default_filter else {}) + } + if target_repo: + return { + "project": { + "name": map_project.name, + "description": map_project.description, + "domain": "General Medical Terminology" + }, + "target_repository": { + "name": target_repo.mnemonic, + "version": target_repo.version, + "filters": project_filters or "Active concepts" + }, + "matching_config": { + "algorithms": ["Fuzzy String", "Semantic Vector", "Lexical"], + "fields_mapped": map_project.fields_mapped, + "thresholds": map_project.score_configuration or {} + }, + "quality_requirements": { + "minimum_confidence": "70%", + "require_exact_class_match": False, + "prefer_active_concepts": True + } + } + return False + + + def __call_anthropic(self, message): # pragma: no cover + return completion(model=self.ANTHROPIC_MODEL, messages=[{'content': message, 'role': 'user'}]) diff --git a/core/settings.py b/core/settings.py index 82dc58dcf..968dca4e4 100644 --- a/core/settings.py +++ b/core/settings.py @@ -609,3 +609,5 @@ if ENV not in ['ci', 'staging', 'production', 'demo']: LM_MODEL_NAME = 'all-MiniLM-L6-v2' LM = SentenceTransformer(LM_MODEL_NAME) + +ANTHROPIC_API_KEY = os.environ.get('ANTHROPIC_API_KEY', None) diff --git a/core/sources/models.py b/core/sources/models.py index c36e6892b..038d31699 100644 --- a/core/sources/models.py +++ b/core/sources/models.py @@ -281,6 +281,24 @@ def concept_filter_order(self): def concept_filter_default(self): return get(self.meta, 'display.default_filter') or None + @property + def filters_ordered(self): + if not self.filters: + return [] + + ordered = [] + ordered_code = [ + *(self.concept_filter_order or []), + *sorted([ + f['code'] for f in self.filters if f.get('code', None) and f['code'] not in self.concept_filter_order + ], key=str.lower) + ] + for code in ordered_code: + filter_obj = next((f for f in self.filters if f.get('code') == code), None) + if filter_obj: + ordered.append(filter_obj) + return ordered + def clean_properties(self): if not self.properties: self.properties = [] diff --git a/core/sources/serializers.py b/core/sources/serializers.py index 647d9d815..a27541c10 100644 --- a/core/sources/serializers.py +++ b/core/sources/serializers.py @@ -363,6 +363,7 @@ class SourceDetailSerializer(SourceCreateOrUpdateSerializer, AbstractRepoResourc client_configs = SerializerMethodField() hierarchy_root = SerializerMethodField() hierarchy_root_url = CharField(source='hierarchy_root.url', required=False, allow_blank=True, allow_null=True) + filters = ListField(source='filters_ordered', allow_null=True) class Meta: model = Source @@ -454,6 +455,7 @@ class SourceVersionDetailSerializer(SourceCreateOrUpdateSerializer, AbstractRepo states = SerializerMethodField() tasks = SerializerMethodField() hierarchy_root_url = CharField(source='hierarchy_root.url', required=False, allow_blank=True, allow_null=True) + filters = ListField(source='filters_ordered', allow_null=True) class Meta: model = Source diff --git a/core/users/constants.py b/core/users/constants.py index 45aa906fd..b0c59cd1f 100644 --- a/core/users/constants.py +++ b/core/users/constants.py @@ -9,5 +9,12 @@ OCL_FHIR_SERVERS_GROUP = 'ocl_fhir_servers' HAPI_FHIR_SERVERS_GROUP = 'hapi_fhir_servers' OPERATIONS_PANEL_GROUP = 'operations_panel' -AUTH_GROUPS = [OCL_SERVERS_GROUP, OCL_FHIR_SERVERS_GROUP, HAPI_FHIR_SERVERS_GROUP, OPERATIONS_PANEL_GROUP] +MAPPER_AI_ASSISTANT_GROUP = 'mapper_ai_assistant' +AUTH_GROUPS = [ + OCL_SERVERS_GROUP, + OCL_FHIR_SERVERS_GROUP, + HAPI_FHIR_SERVERS_GROUP, + OPERATIONS_PANEL_GROUP, + MAPPER_AI_ASSISTANT_GROUP +] INVALID_AUTH_GROUP_NAME = 'Invalid auth group.' diff --git a/core/users/serializers.py b/core/users/serializers.py index 98552a20b..f4f93eb27 100644 --- a/core/users/serializers.py +++ b/core/users/serializers.py @@ -264,3 +264,16 @@ def update(self, instance, validated_data): if instance.id: instance.set_checksums() return instance + + +class UserContentSummarySerializer(serializers.Serializer): # pylint: disable=abstract-method + username = serializers.CharField(read_only=True) + concepts_created = IntegerField(read_only=True) + concepts_updated = IntegerField(read_only=True) + mappings_created = IntegerField(read_only=True) + mappings_updated = IntegerField(read_only=True) + sources_owned = IntegerField(read_only=True) + collections_owned = IntegerField(read_only=True) + references_added = IntegerField(read_only=True) + versions_created = IntegerField(read_only=True) + expansions_created = IntegerField(read_only=True) diff --git a/core/users/tests/tests.py b/core/users/tests/tests.py index dfb769f16..1338f5cd8 100644 --- a/core/users/tests/tests.py +++ b/core/users/tests/tests.py @@ -382,3 +382,80 @@ def test_send_user_reset_password_email(self, send_mail_mock): self.assertTrue(user.reset_password_url in mail.body) self.assertTrue(f'Hi {user.username},' in mail.body) send_mail_mock.assert_called_once() + + +class UserContentSummaryViewTest(OCLAPITestCase): + def setUp(self): + super().setUp() + self.user = UserProfileFactory(username='summaryuser') + self.token = self.user.get_token() + + def test_unauthenticated(self): + response = self.client.get('/users/summaryuser/content-summary/') + self.assertEqual(response.status_code, 401) + + def test_self_user(self): + response = self.client.get( + '/users/summaryuser/content-summary/', + HTTP_AUTHORIZATION='Token ' + self.token, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['username'], 'summaryuser') + self.assertEqual(response.data['concepts_created'], 0) + self.assertEqual(response.data['concepts_updated'], 0) + self.assertEqual(response.data['mappings_created'], 0) + self.assertEqual(response.data['mappings_updated'], 0) + self.assertEqual(response.data['sources_owned'], 0) + self.assertEqual(response.data['collections_owned'], 0) + self.assertEqual(response.data['references_added'], 0) + self.assertEqual(response.data['versions_created'], 0) + self.assertEqual(response.data['expansions_created'], 0) + + def test_other_user_forbidden(self): + other_user = UserProfileFactory(username='otheruser') + other_token = other_user.get_token() + + response = self.client.get( + '/users/summaryuser/content-summary/', + HTTP_AUTHORIZATION='Token ' + other_token, + ) + self.assertEqual(response.status_code, 403) + + def test_staff_can_view_other_user(self): + admin = UserProfileFactory(username='adminuser', is_staff=True) + admin_token = admin.get_token() + + response = self.client.get( + '/users/summaryuser/content-summary/', + HTTP_AUTHORIZATION='Token ' + admin_token, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['username'], 'summaryuser') + + def test_nonexistent_user(self): + response = self.client.get( + '/users/doesnotexist/content-summary/', + HTTP_AUTHORIZATION='Token ' + self.token, + ) + self.assertEqual(response.status_code, 404) + + def test_with_content(self): + from core.sources.tests.factories import UserSourceFactory + from core.concepts.tests.factories import ConceptFactory + from core.mappings.tests.factories import MappingFactory + from core.collections.tests.factories import UserCollectionFactory + + source = UserSourceFactory(user=self.user, created_by=self.user) + ConceptFactory(parent=source, created_by=self.user, updated_by=self.user) + MappingFactory(parent=source, created_by=self.user, updated_by=self.user) + UserCollectionFactory(user=self.user, created_by=self.user) + + response = self.client.get( + '/users/summaryuser/content-summary/', + HTTP_AUTHORIZATION='Token ' + self.token, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data['sources_owned'], 1) + self.assertEqual(response.data['collections_owned'], 1) + self.assertGreaterEqual(response.data['concepts_created'], 1) + self.assertGreaterEqual(response.data['mappings_created'], 1) diff --git a/core/users/urls.py b/core/users/urls.py index aa56a16b2..603634048 100644 --- a/core/users/urls.py +++ b/core/users/urls.py @@ -30,6 +30,7 @@ path('/following//', views.UserFollowingView.as_view(), name='userprofile-following'), path('/orgs/', org_views.OrganizationListView.as_view(), name='userprofile-orgs'), path('/extras/', views.UserExtrasView.as_view(), name='user-extras'), + path('/content-summary/', views.UserContentSummaryView.as_view(), name='user-content-summary'), path( '/orgs/sources/', org_views.OrganizationSourceListView.as_view(), name='userprofile-organization-source-list'), diff --git a/core/users/views.py b/core/users/views.py index bd27ea2cf..c968785f4 100644 --- a/core/users/views.py +++ b/core/users/views.py @@ -19,7 +19,7 @@ from rest_framework.response import Response from rest_framework.views import APIView -from core.common.constants import NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY, LAST_LOGIN_SINCE_PARAM, \ +from core.common.constants import NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY, HEAD, LAST_LOGIN_SINCE_PARAM, \ LAST_LOGIN_BEFORE_PARAM, DATE_JOINED_SINCE_PARAM, DATE_JOINED_BEFORE_PARAM, UPDATED_BY_USERNAME_PARAM from core.common.exceptions import Http400 from core.common.mixins import ListWithHeadersMixin @@ -35,7 +35,7 @@ from core.users.documents import UserProfileDocument from core.users.search import UserProfileFacetedSearch from core.users.serializers import UserDetailSerializer, UserCreateSerializer, UserListSerializer, \ - UserSummarySerializer, FollowingSerializer + UserSummarySerializer, FollowingSerializer, UserContentSummarySerializer from .models import UserProfile, Follow, UserRateLimit from ..common import ERRBIT_LOGGER from ..common.throttling import ThrottleUtil @@ -635,3 +635,51 @@ def perform_destroy(self, instance): follower.unfollow(instance.following) return Response(status=status.HTTP_204_NO_CONTENT) + + +class UserContentSummaryView(APIView): + permission_classes = (IsAuthenticated,) + serializer_class = UserContentSummarySerializer + + def get(self, request, user): + from django.db.models import F + from core.collections.models import Collection, CollectionReference, Expansion + from core.concepts.models import Concept + from core.mappings.models import Mapping + from core.sources.models import Source + + try: + profile = UserProfile.objects.get(username=user) + except UserProfile.DoesNotExist: + return Response(status=status.HTTP_404_NOT_FOUND) + + is_self = request.user.username == profile.username + if not is_self and not request.user.is_staff: + raise PermissionDenied() + + head_concepts = Concept.objects.filter(id=F('versioned_object_id')) + head_mappings = Mapping.objects.filter(id=F('versioned_object_id')) + user_head_sources = Source.objects.filter(user=profile, version=HEAD) + user_head_collections = Collection.objects.filter(user=profile, version=HEAD) + + # Non-HEAD versions created by this user (version releases) + user_source_versions = Source.objects.filter( + created_by=profile).exclude(version=HEAD) + user_collection_versions = Collection.objects.filter( + created_by=profile).exclude(version=HEAD) + + data = { + 'username': profile.username, + 'concepts_created': head_concepts.filter(created_by=profile).count(), + 'concepts_updated': head_concepts.filter(updated_by=profile).count(), + 'mappings_created': head_mappings.filter(created_by=profile).count(), + 'mappings_updated': head_mappings.filter(updated_by=profile).count(), + 'sources_owned': user_head_sources.count(), + 'collections_owned': user_head_collections.count(), + 'references_added': CollectionReference.objects.filter( + collection__in=user_head_collections, created_by=profile).count(), + 'versions_created': user_source_versions.count() + user_collection_versions.count(), + 'expansions_created': Expansion.objects.filter(created_by=profile).count(), + } + serializer = self.serializer_class(data) + return Response(serializer.data) diff --git a/docker-compose.yml b/docker-compose.yml index 8cdcbf418..ac2d13550 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,6 +60,7 @@ services: - MINIO_SECRET_KEY - MINIO_SECURE - ENABLE_THROTTLING + - ANTHROPIC_API_KEY healthcheck: test: "curl --silent --fail http://localhost:8000/version/ || exit 1" volumes: diff --git a/requirements.txt b/requirements.txt index 38dd9d3e8..45853f6ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,4 +45,5 @@ responses==0.25.0 pydantic==2.6.4 minio==7.2.9 sentence-transformers==3.3.1 -elastic-transport==8.17 \ No newline at end of file +elastic-transport==8.17 +litellm==1.77.0 \ No newline at end of file