Coverage for apis_core/generic/importers.py: 0%

1import json 

2import logging 

3import urllib 

4from functools import cache 

5 

6from django.core.exceptions import ImproperlyConfigured 

7 

8from apis_core.utils.normalize import clean_uri 

9from apis_core.utils.rdf import get_definition_and_attributes_from_uri 

10 

11logger = logging.getLogger(__name__) 

12 

13 

14class GenericModelImporter: 

15 """ 

16 A generic importer class 

17 It provides the standard methods for importing data from 

18 an URI and creating a model instance of it. 

19 By default it fetches a resource, first tries to parse it using 

20 our rdf parser, if that fails tries to parse it using json and 

21 then extracts the fields whose keys match the model field names. 

22 Projects can inherit from this class and override the default 

23 methods or simple write their own from scratch. 

24 """ 

25 

26 model = None 

27 import_uri = None 

28 

29 def __init__(self, uri, model): 

30 self.model = model 

31 self.import_uri = self.clean_uri(uri) 

32 

33 @property 

34 def get_uri(self): 

35 return self.import_uri 

36 

37 def clean_uri(self, uri): 

38 return clean_uri(uri) 

39 

40 @cache 

41 def request(self, uri): 

42 # we first try to use the RDF parser 

43 try: 

44 defn, data = get_definition_and_attributes_from_uri(uri, self.model) 

45 return data 

46 except Exception as e: 

47 logger.debug(e) 

48 # if everything else fails, try parsing json 

49 # if even that does not help, return an empty dict 

50 try: 

51 return json.loads(urllib.request.urlopen(uri).read()) 

52 except Exception as e: 

53 logger.debug(e) 

54 return {} 

55 

56 def mangle_data(self, data): 

57 return data 

58 

59 def get_data(self, drop_unknown_fields=True): 

60 """ 

61 fetch the data using the `request` method and 

62 mangle the data using the `mangle_data` method. 

63 If the `drop_unknown_fields` argument is true, 

64 remove all fields from the data dict that do not 

65 have an equivalent field in the model. 

66 """ 

67 data = self.request(self.import_uri) 

68 data = self.mangle_data(data) 

69 if drop_unknown_fields: 

70 # we are dropping all fields that are not part of the model 

71 modelfields = [field.name for field in self.model._meta.fields] 

72 data = {key: data[key] for key in data if key in modelfields} 

73 if not data: 

74 raise ImproperlyConfigured( 

75 f"Could not import {self.import_uri}. Data fetched was: {data}" 

76 ) 

77 return data 

78 

79 def import_into_instance(self, instance, fields="__all__"): 

80 data = self.get_data() 

81 if fields == "__all__": 

82 fields = data.keys() 

83 for field in fields: 

84 if hasattr(instance, field) and field in data.keys(): 

85 setattr(instance, field, data[field]) 

86 instance.save() 

87 

88 def create_instance(self): 

89 return self.model.objects.create(**self.get_data(drop_unknown_fields=True))