oort.rdfview

1 # -*- coding: UTF-8 -*- 2 """This module contains a system for creating rdf query classes in a mainly 3 declarative manner. These are built by subclassing ``RdfQuery`` and creating 4 class attributes by using ``Selector``:s. There are several selectors provided 5 in this module to cover all regular cases of data acquisition. 6 """ 7 #======================================================================= 8 from itertools import chain 9 from types import ModuleType 10 import warnings 11 from rdflib import RDF, RDFS, Namespace, URIRef, BNode, Literal 12 from rdflib import ConjunctiveGraph 13 try: import simplejson 14 except ImportError: simplejson = None 15 from oort.util.code import contract 16 #======================================================================= 17 18 19 # TODO: deprecate THIS_QUERY and then remove this? 20 THIS_QUERY = object() 21 22 MODULE_SEP = ':' 23 24 25 #----------------------------------------------------------------------- 26 27

28 -class Selector(object):

29 __slots__ = ('predicate', '_namespace', 'filters', 30 '_subQueryMarker', '_finalSubQuery', 31 '_name', '_queryClass') 32

33 - def __init__(self, predBase=None, subQuery=None):

34 self.predicate = None 35 self._namespace = None 36 if isinstance(predBase, Namespace) or isinstance(predBase, ModuleType): 37 self._namespace = predBase 38 elif isinstance(predBase, URIRef): 39 self.predicate = predBase 40 self._subQueryMarker = subQuery 41 self._finalSubQuery = False 42 self.filters = []

43 44 @contract.state_change

45 - def hook_into_rdf_query(self, name, queryClass):

46 self._name = name 47 self._queryClass = queryClass 48 if not self._namespace: 49 self._namespace = queryClass._namespace 50 if not self.predicate and self._namespace: 51 if isinstance(self._namespace, ModuleType): 52 self.predicate = getattr(self._namespace, name) 53 else: 54 self.predicate = self._namespace[name]

55 # TODO: not used since e.g. the selector decorator uses no predicate 56 #if not self.predicate: 57 # raise ValueError( 58 # "Could not determine predicate for Selector %s" % self) 59

60 - def get_sub_query(self):

61 final = self._finalSubQuery 62 if final is False: 63 marker = self._subQueryMarker 64 final = marker 65 if isinstance(marker, basestring): 66 if MODULE_SEP in marker: 67 module, marker = marker.split(MODULE_SEP) 68 else: 69 module = self._queryClass.__module__ 70 final = __import__(module, fromlist=['']).__dict__[marker] 71 elif marker is THIS_QUERY: 72 final = self._queryClass 73 self._finalSubQuery = final 74 return final

75

76 - def __get__(self, rdfQueryInstance, rdfQueryOwnerClass=None):

77 if not rdfQueryInstance: 78 return self 79 prep = rdfQueryInstance._preparedSelects[self._name] 80 if not prep.hasRun: 81 result = self.retreive_result(rdfQueryInstance, prep.selectArgs) 82 for fltr in self.filters: 83 result = fltr(result) 84 prep.hasRun = True 85 prep.result = result 86 return prep.result

87 88 @contract.default_method

89 - def retreive_result(self, rdfQueryInstance, selectArgs):

90 result = self._process_for_subqueries( 91 rdfQueryInstance, 92 self.select(*selectArgs), 93 *selectArgs[:2] 94 ) 95 return result

96

97 - def _process_for_subqueries(self, rdfQueryInstance, 98 rawresults, graph, lang):

99 returnList = isinstance(rawresults, list) 100 if not rawresults: 101 if returnList: return [] 102 else: return None 103 104 subQuery = self.get_sub_query() 105 if not subQuery: 106 return rawresults 107 else: 108 # TODO: using THIS_QUERY (and no execCache?) may currently cause 109 # infinite loops..? But things are more lazy now; perhaps not.. 110 run_query = query_or_cached(subQuery, rdfQueryInstance._execCache) 111 if returnList: 112 return [run_query(graph, lang, uri) for uri in rawresults] 113 else: 114 return run_query(graph, lang, rawresults)

115 116 @contract.template_method

117 - def select(self, graph, lang, subject):

118 raise NotImplementedError 119 return None or []

120 121 @contract.default_method

122 - def back_to_graph(self, graph, subject, value):

123 pass

124

125 - def viewed_as(self, subQuery):

126 self._subQueryMarker = subQuery 127 return self

128

129 - def __rshift__(self, subQuery):

130 return self.viewed_as(subQuery)

131

132 - def add_filter(self, fltr):

133 self.filters.append(fltr)

134

135 - def __or__(self, fltr):

136 self.add_filter(fltr) 137 return self

138

139 - def __set__(self, rdfQueryInstance, value):

140 prep = rdfQueryInstance._preparedSelects[self._name] 141 lang = rdfQueryInstance._lang 142 sub = self.get_sub_query() 143 if isinstance(value, list): 144 value = [self.type_raw_value(val, lang) for val in value] 145 else: 146 value = self.type_raw_value(value, lang) 147 if sub: 148 if isinstance(value, list): 149 prep.result = [sub.from_dict(val, lang, BNode()) 150 for val in value] 151 else: 152 prep.result = sub.from_dict(value, lang, BNode()) 153 else: 154 prep.result = value 155 prep.hasRun = True

156 157 @contract.default_method

158 - def type_raw_value(self, value, lang):

159 # TODO: need more clever type mapping. also, allow {'_uri': ".." } to be resource? 160 if isinstance(value, basestring): 161 value = Literal(value) 162 return value

163 164

165 -class PreparedSelect(object):

166 __slots__ = ('selectArgs', 'result', 'hasRun')

167 - def __init__(self, graph, lang, subject):

168 self.selectArgs = (graph, lang, subject) 169 self.result = None 170 self.hasRun = False # TODO: make configurable?

171 172

173 -class _rdf_query_meta(type):

174 - def __init__(cls, clsName, bases, clsDict):

175 super(_rdf_query_meta, cls).__init__(clsName, bases, clsDict) 176 177 cls._selectors = selectors = {} 178 for base in bases: 179 if hasattr(base, '_selectors'): 180 selectors.update(base._selectors) 181 182 rdfBase = clsDict.get('_rdfbase_') 183 if not rdfBase: 184 for base in bases: 185 if hasattr(base, '_rdfbase_'): 186 rdfBase = base._rdfbase_ 187 break 188 if isinstance(rdfBase, Namespace): 189 cls._namespace = rdfBase 190 if not clsDict.get('RDF_TYPE'): 191 cls.RDF_TYPE = rdfBase[clsName] 192 else: 193 cls._namespace = None # TODO: pick from type? 194 195 for key, value in clsDict.items(): 196 if isinstance(value, Selector): 197 value.hook_into_rdf_query(key, cls) 198 selectors[key] = value

199 200

201 -class RdfQuery(object):

202 __metaclass__ = _rdf_query_meta 203 204 RDF_TYPE = RDFS.Resource 205 206 # TODO: test use of execCache propertly (it seems to work though) 207

208 - def __init__(self, graph, lang, subject, execCache=None):

209 self._graph = graph 210 self._subject = subject 211 self._lang = lang 212 self._preparedSelects = self._make_prepare_selects() 213 self._execCache = execCache

214

215 - def _make_prepare_selects(self):

216 prepareds = {} 217 graph, lang, subject = self._graph, self._lang, self._subject 218 for name, selector in self._selectors.items(): 219 if not subject: 220 # FIXME: happens when subject is a string/Literal - wrong in 221 # itself! Remove or signal error? As it is, it leads to 222 # illegible errors further down! 223 # Also, why not: if subject == u'': 224 #setattr(self, name, None) # TODO:removed; see this fixme 225 continue 226 prep = PreparedSelect(graph, lang, subject) 227 prepareds[name] = prep 228 return prepareds

229

230 - def __str__(self):

231 return str(self._subject)

232

233 - def __eq__(self, other):

234 if isinstance(other, RdfQuery): 235 return self._subject == other._subject 236 else: 237 return self._subject == other

238 239 @classmethod

240 - def bound_with(cls, subject, lang=None):

241 def bound_query(graph, _lang, _subject): 242 return cls(graph, lang or _lang, subject)

243 bound_query.query = cls 244 bound_query.__name__ = cls.__name__ 245 return bound_query

246 247 @classmethod

248 - def from_dict(cls, data, lang, subject):

249 graph = ConjunctiveGraph() 250 query = cls(graph, lang, subject) 251 for k, v in data.items(): 252 setattr(query, k, v) 253 return query

254 255 @classmethod

256 - def find_by(cls, graph, lang, execCache=None, **kwargs):

257 assert len(kwargs) == 1 258 name, value = kwargs.items()[0] 259 predicate = cls._selectors[name].predicate 260 for subject in graph.subjects(predicate, value): 261 yield query_or_cached(cls, execCache)(graph, lang, subject)

262 263 @property

264 - def uri(self):

265 return self._subject

266

267 - def get_selected_value(self, name):

268 return self._preparedSelects[name].result

269

270 - def to_graph(self, newgraph=None):

271 subject = self._subject or BNode() # FIXME: is this ok? 272 if not subject: return # FIXME, see fixme in __init__ 273 274 lgraph = newgraph or ConjunctiveGraph() 275 if not newgraph: 276 for key, ns in self._graph.namespaces(): 277 lgraph.bind(key, ns) 278 279 for t in self._graph.objects(subject, RDF.type): 280 lgraph.add((subject, RDF.type, t)) 281 282 for selector in self._selectors.values(): 283 value = selector.__get__(self) 284 if not value: 285 continue 286 selector.back_to_graph(lgraph, subject, value) 287 288 # FIXME: why is this happening; how can we prevent it? 289 for t in lgraph: 290 if None in t: lgraph.remove(t) 291 return lgraph

292

293 - def to_rdf(self):

294 return self.to_graph().serialize(format='pretty-xml')

295

296 - def to_dict(self, keepSubject=False):

297 d = {} 298 if keepSubject: 299 # TODO: sync with new property 'uri' 300 subjectKey = isinstance(keepSubject, str) and keepSubject or 'resource' 301 subj = self._subject 302 if subj and not isinstance(subj, BNode): 303 d[subjectKey] = self._subject 304 305 for selector in self._selectors.values(): 306 name = selector._name 307 value = selector.__get__(self) 308 if not value: 309 continue 310 if isinstance(value, dict): 311 d[name] = dict([(key, self.__dict_convert(val, keepSubject)) 312 for key, val in value.items()]) 313 elif hasattr(value, '__iter__'): 314 d[name] = [self.__dict_convert(val, keepSubject) 315 for val in value] 316 else: 317 d[name] = self.__dict_convert(value, keepSubject) 318 # TODO: handle xml literals 319 return d

320

321 - def __dict_convert(self, value, keepSubject):

322 if isinstance(value, RdfQuery): 323 return value.to_dict(keepSubject) 324 else: 325 return unicode(value) # TODO: simple type conversions?

326

327 - def to_json(self, keepSubject=False):

328 if simplejson: 329 return simplejson.dumps(self.to_dict(keepSubject)) 330 else: 331 raise NotImplementedError

332 333 334 #----------------------------------------------------------------------- 335 336 337 # Is the use of weakref fine enough (reasonably needed to avoid cyclic 338 # references and hence possible memory leaks)? 339 # See: <http://docs.python.org/lib/module-weakref.html> 340 from weakref import WeakValueDictionary 341

342 -class ExecCache(object):

343 """ 344 This is a query execution cache which reuses results for the same query, 345 subject and language, avoiding multiple instances of the same query when 346 given the same subject and lang. 347 """

348 - def __init__(self):

349 self.cache = WeakValueDictionary()

350 - def __call__(self, query, graph, lang, subject):

351 cache = self.cache 352 key = (id(query), unicode(subject), lang) 353 #key = (query, subject, lang) 354 result = cache.get(key) 355 if not result: 356 result = query(graph, lang, subject, self) 357 cache[key] = result 358 return result

359 360

361 -def query_or_cached(rdfQuery, execCache):

362 if execCache: 363 def run_query(graph, lang, uri): 364 return execCache(rdfQuery, graph, lang, uri)

365 return run_query 366 else: 367 return rdfQuery 368 369

370 -def run_queries(queries, graph, lang, subject):

371 execCache = ExecCache() 372 for query in queries: 373 yield execCache(query, graph, lang, subject)

374 375 376 #----------------------------------------------------------------------- 377 378

379 -class Filter(object):

380 - def __init__(self, func):

381 self.func = func

382 - def __call__(self, items):

383 return filter(self.func, items)

384 385

386 -class Sorter(object):

387 - def __init__(self, obj=None, reverse=False, ignoreCase=False):

388 if callable(obj): 389 self.attr = None 390 self.func = obj 391 else: 392 self.attr = obj 393 self.func = None 394 self.reverse = reverse 395 self.ignoreCase = ignoreCase

396 - def __call__(self, items):

397 copy = items[:] 398 copy.sort(self.sort) 399 if self.reverse: 400 copy.reverse() 401 return copy

402 - def sort(self, r1, r2):

403 attr = self.attr 404 func = self.func 405 if attr: 406 v1, v2 = getattr(r1, attr, r1), getattr(r2, attr, r2) 407 elif func: 408 v1, v2 = func(r1), func(r2) 409 else: 410 v1, v2 = r1, r2 411 if self.ignoreCase: 412 v1, v2 = v1.lower(), v2.lower() 413 return cmp(v1, v2)

414 415 416 #----------------------------------------------------------------------- 417 418 419 # TODO: totally untested! 420 # - use: TypeSwitch(persons=Person, values=Literal) 421 # - a subclass of RdfQuery? Or affect selector..? Reasonably yes.. 422 # - should adapt to if stuff is a list or one thing (one or each) 423 # - how about localized? 424 # - also should be used as list *or*: 425 # - obj.switchedstuff.persons 426 #def type_switch(typeSelectors, default): 427 # rdfType = graph.value(resource, RDF.type, None, any=True) 428 # def select(graph, lang, resource, **kwargs): 429 # query = typeSelectors.get(rdfType, default) 430 # return query(graph, lang, resource, **kwargs) 431 # return select 432 433 434

435 -def back_from_value(graph, subject, predicate, value):

436 if isinstance(value, RdfQuery): 437 graph.add((subject, predicate, value._subject)) 438 value.to_graph(graph) 439 else: 440 if not isinstance(value, list): # TODO: fix this 441 graph.add((subject, predicate, value))

442 443

444 -class UnarySelector(Selector):

445 - def back_to_graph(self, graph, subject, value):

446 back_from_value(graph, subject, self.predicate, value)

447

448 -class EachSelector(Selector):

449 - def back_to_graph(self, graph, subject, values):

450 for value in values: 451 back_from_value(graph, subject, self.predicate, value)

452 453

454 -class one(UnarySelector):

455 - def select(self, graph, lang, subject):

456 return graph.value(subject, self.predicate, None, any=True)

457 458

459 -class each(EachSelector):

460 - def select(self, graph, lang, subject):

461 return list(graph.objects(subject, self.predicate))

462 463

464 -class one_where_self_is(Selector):

465 - def select(self, graph, lang, subject):

466 return graph.value(None, self.predicate, subject, any=True)

467

468 - def back_to_graph(self, graph, subject, value):

469 back_from_value(graph, value._subject, self.predicate, subject)

470 471

472 -class each_where_self_is(Selector):

473 - def select(self, graph, lang, subject):

474 return list(graph.subjects(self.predicate, subject))

475

476 - def back_to_graph(self, graph, subject, values):

477 for value in values: 478 back_from_value(graph, value._subject, self.predicate, subject)

479 480

481 -class collection(Selector):

482 - def __init__(self, predBase=None, subQuery=None, multiple=False):

483 Selector.__init__(self, predBase, subQuery) 484 self.multiple = multiple

485

486 - def select(self, graph, lang, subject):

487 if self.multiple: 488 allItems = [graph.items(res) 489 for res in graph.objects(subject, self.predicate)] 490 return list(chain(*allItems)) 491 else: 492 return list(graph.items( 493 graph.value(subject, self.predicate, None, any=True) 494 ))

495

496 - def back_to_graph(self, graph, subject, values):

497 if not values: 498 graph.add((subject, self.predicate, RDF.nil)) 499 return 500 bnode = BNode() 501 graph.add((subject, self.predicate, bnode)) 502 for value in values: 503 back_from_value(graph, bnode, RDF.first, value) 504 newBnode = BNode() 505 graph.add((bnode, RDF.rest, newBnode)) 506 bnode = newBnode 507 graph.add((bnode, RDF.rest, RDF.nil))

508 509

510 -class TypeLocalized(Selector):

511 - def type_raw_value(self, value, lang):

512 if isinstance(value, basestring): 513 value = Literal(value, lang) 514 return value

515

516 -class localized(TypeLocalized, UnarySelector):

517 - def select(self, graph, lang, subject):

518 first = None 519 for value in graph.objects(subject, self.predicate): 520 if not first: first = value 521 if getattr(value, 'language', None) == lang: 522 return value 523 return first

524 525 526 # TODO: This is a hackish solution; see also below (transparently using datatype). 527 # It also reduces the literal, making it irreversible (should store original value!). 528 try: 529 from oort.util._genshifilters import language_filtered_xml 530 except ImportError, e: 531 warnings.warn("Could not import _genshifilters. Error was: %r. The selector 'localized_xml' will not be available." % e) 532 else:

533 - class localized_xml(UnarySelector):

534 """This selector removes any elements with an xml:lang other than the 535 current language. It also supports the never standardized 'rdf-wrapper' in 536 XML Literals, who are removed from the output. 537 538 Important! This is currently tied to the Genshi Templating System, and may 539 not work as expected in all cases.""" 540

541 - def select(self, graph, lang, subject):

542 return language_filtered_xml( 543 graph.objects(subject, self.predicate), lang)

544

545 - def type_raw_value(self, value, lang):

546 if isinstance(value, basestring): 547 value = Literal(value, datatype=RDF.XMLLiteral) 548 return value

549

550 -class i18n_dict(Selector):

551 - def select(self, graph, lang, subject):

552 valueDict = {} 553 for value in graph.objects(subject, self.predicate): 554 valueDict[value.language] = value 555 return valueDict

556

557 - def back_to_graph(self, graph, subject, value):

558 for lang, text in value.items(): 559 graph.add((subject, self.predicate, Literal(text, lang=lang)))

560 561

562 -class each_localized(TypeLocalized, EachSelector):

563 - def select(self, graph, lang, subject):

564 return [ value for value in graph.objects(subject, self.predicate) 565 if value.language == lang ]

566 567 568 #----------------------------------------------------------------------- 569 570 571 # TODO: Though "widely used" (by me), I think this was a little premature. 572 # There seems little use for this that a regular property can't do (getting the 573 # graph, lang and subject from self -- where needed). Perhaps I should include 574 # a memoized codeutil so it's easy to create lazily calculated bigger things. 575 # 576 # Even worse, this "utility" bypasses _process_for_subqueries, which is 577 # intricate and very close to the implementation. And this is the only reason 578 # retreive_result is marked as a "default_method"; it should reasonably be 579 # private. 580 #

581 -class selector(Selector):

582 "Use as decorator for methods of an RdfQuery subclass to convert them to selectors."

583 - def __init__(self, func):

584 super(selector, self).__init__(None) 585 self.func = func

586 - def retreive_result(self, rdfQueryInstance, selectArgs):

587 return self.func(rdfQueryInstance, *selectArgs)

588 @classmethod

589 - def filtered_by(cls, *filters):

590 def decorator(func): 591 sel = cls(func) 592 for fltr in filters: 593 sel.add_filter(fltr) 594 return sel

595 return decorator

596 597 598 #----------------------------------------------------------------------- 599 # TODO: consider returning ElementTree data for XML Literals. And if so, also 600 # filtered on u'{http://www.w3.org/XML/1998/namespace}lang' (keep if none) for localized. 601 602 # TODO: also consider checking datatype and coercing (at least) these: 603 # Use: rdflib.Literal.castPythonToLiteral 604 # See: rdflib.sparql.sparqlOperators.getLiteralValue(v) 605 # See: <http://en.wikipedia.org/wiki/RDFLib#RDF_Literal_Support> 606 # - NOTE: Isn't this done automatically by rdflib now? I believe so. 607 608 609 #----------------------------------------------------------------------- 610 611

612 -class QueryContext(object):

613 """ 614 A query context, used to provide a managed context for query execution. 615 616 Initalized with: 617 618 - graph 619 - language or getter for language 620 - a set of queries or a modules containing queries 621 Accessible as attributes on the context or via view_for using RDF_TYPE 622 623 """ 624

625 - def __init__(self, graph, langobj, queries=None, query_modules=None):

626 self._graph = graph 627 self._execCache = ExecCache() 628 629 if callable(langobj): 630 get_lang = langobj 631 else: 632 def get_lang(): return langobj 633 self._get_lang = get_lang 634 635 self._querydict = querydict = {} 636 if queries: 637 for query in queries: 638 querydict[query.__name__] = query 639 if query_modules: 640 for module in query_modules: 641 for name, obj in module.__dict__.items(): 642 if isinstance(obj, type) and issubclass(obj, RdfQuery): 643 querydict[name] = obj 644 645 self._queryTypeMap = {} 646 for query in querydict.values(): 647 self._queryTypeMap[query.RDF_TYPE] = query

648

649 - def __getattr__(self, name):

650 try: 651 query = self._querydict[name] 652 return self._prepared_query(query) 653 except KeyError: 654 raise AttributeError("%s has no attribute '%s'" % (self, name))

655

656 - def view_for(self, uriref):

657 for typeref in self._graph.objects(uriref, RDF.type): 658 query = self._queryTypeMap.get(typeref) 659 if query: 660 return self._prepared_query(query)(uriref) 661 raise KeyError("%s has no query for type '%s'" % (self, uriref))

662

663 - def _prepared_query(self, query):

664 return self.PreparedQuery(self, query)

665

666 - class PreparedQuery(object):

667 __slots__ = ('query', 'context') 668

669 - def __init__(self, context, query):

670 self.context = context 671 self.query = query

672

673 - def __call__(self, subject):

674 cx = self.context 675 return cx._execCache(self.query, cx._graph, cx._get_lang(), subject)

676

677 - def find_all(self):

678 cx = self.context 679 for subject in cx._graph.subjects(RDF.type, self.query.RDF_TYPE): 680 yield cx.view_for(subject)

681

682 - def find_by(self, **kwargs):

683 cx = self.context 684 return self.query.find_by(cx._graph, cx._get_lang(), 685 execCache=cx._execCache, **kwargs)

686

Source Code for Module oort.rdfview