'''
Created on 2021-01-29
@author: wf
'''
import unittest
import copy
import io
import os
from contextlib import redirect_stdout
from lodstorage.query import QueryManager, Query, QueryResultDocumentation, EndpointManager, ValueFormatter, Format
from lodstorage.querymain import main as queryMain
from lodstorage.querymain import QueryMain
from lodstorage.sparql import SPARQL
import tests.testSqlite3
from tests.basetest import Basetest
[docs]class TestQueries(Basetest):
'''
Test query handling
'''
[docs] def testSQLQueries(self):
'''
see https://github.com/WolfgangFahl/pyLoDStorage/issues/19
'''
show=self.debug
qm=QueryManager(lang='sql',debug=False)
self.assertEqual(2,len(qm.queriesByName))
sqlDB=tests.testSqlite3.TestSQLDB.getSampleTableDB()
#print(sqlDB.getTableDict())
for _name,query in qm.queriesByName.items():
listOfDicts=sqlDB.query(query.query)
resultDoc=query.documentQueryResult(listOfDicts)
if show:
print(resultDoc)
pass
[docs] def testSparqlQueries(self):
'''
test SPARQL queries
'''
show=self.debug
show=True
qm=QueryManager(lang='sparql',debug=False)
for name,query in qm.queriesByName.items():
if name in ["US President Nicknames"]:
if show:
print(f"{name}:{query}")
endpoint=SPARQL(query.endpoint)
try:
qlod=endpoint.queryAsListOfDicts(query.query)
for tablefmt in ["mediawiki","github","latex"]:
doc=query.documentQueryResult(qlod, tablefmt=tablefmt,floatfmt=".0f")
docstr=doc.asText()
if show:
print (docstr)
except Exception as ex:
print(f"{query.title} at {query.endpoint} failed: {ex}")
[docs] def testUnicode2LatexWorkaround(self):
'''
test the uniCode2Latex conversion workaround
'''
debug=self.debug
for code in range(8320,8330):
uc=chr(code)
latex=QueryResultDocumentation.uniCode2Latex(uc)
if debug:
print(f"{uc}→{latex}")
#self.assertTrue(latex.startswith("$_"))
unicode="À votre santé!"
latex=QueryResultDocumentation.uniCode2Latex(unicode,withConvert=True)
if debug:
print(f"{unicode}→{latex}")
self.assertEqual("\\`A votre sant\\'e!",latex)
[docs] def captureQueryMain(self,args):
'''
run queryMain and capture stdout
Args:
args(list): command line arguments
Returns:
str: the stdout content of the command line call
'''
stdout = io.StringIO()
with redirect_stdout(stdout):
queryMain(args)
result = stdout.getvalue()
return result
[docs] def testQueryCommandLine(self):
'''
test the sparql query command line
'''
debug=self.debug
#debug=True
for testArg in [
{"format":"csv", "expected":'''Theodore Roosevelt","Teddy"'''},
{"format":"xml", "expected":'''<?xml version="1.0" ?>'''},
{"format":"latex","expected":'''Theodore Roosevelt & Teddy'''},
{"format":"mediawiki", "expected":'''| [https://www.wikidata.org/wiki/Q33866 Q33866] || Theodore Roosevelt || Teddy'''},
{"format":"github","expected":'''| [Q33866](https://www.wikidata.org/wiki/Q33866) | Theodore Roosevelt | Teddy |'''}
]:
resultFormat=testArg["format"]
expected=testArg["expected"]
args=["-d","-qn","US President Nicknames","-l","sparql","-f",resultFormat]
result=self.captureQueryMain(args)
if debug:
print(f"{resultFormat}:{result}")
self.assertTrue(expected in result)
[docs] def testQueryEndpoints(self):
"""
tests the sparql endpoint commandline endpoint selection
"""
testArgs=[
{"en":"wikidata",},
#{"en":"qlever-wikidata",},
# workaround https://github.com/ad-freiburg/qlever/issues/631
#{"en":"qlever-wikidata-proxy",},
]
debug=self.debug
showServerDown=True
for testArg in testArgs:
endpointName = testArg.get("en")
args = ["-d", "-qn", "cities", "-p","-l", "sparql", "-f", "json", "-en", endpointName, "-raw"]
result=self.captureQueryMain(args)
if not "503 Service Unavailable" in result:
#if debug:
# print(result)
self.assertTrue("Arnis" in result, f"{endpointName}: Arnis not in query result")
elif showServerDown:
print(f"{endpointName} returns 503 Service Unavailable")
[docs] def testIssue69showEndpoints(self):
'''
test the listEndpoints option
https://github.com/WolfgangFahl/pyLoDStorage/issues/69
'''
debug=self.debug
#debug=True
expected={"sparql":"wikidata"}
for option in ["-le","--listEndpoints"]:
for lang in ["sparql","sql"]:
args=[option,"-l",lang]
result=self.captureQueryMain(args)
if debug:
print(result)
if lang in expected:
self.assertTrue(expected[lang] in result)
[docs] def testIssue70showQuery(self):
'''
test the showQuery option
https://github.com/WolfgangFahl/pyLoDStorage/issues/70
'''
for option in ["-sq","--showQuery"]:
args=["-qn","10 Largest Cities Of The World",option,"-l","sparql"]
result=self.captureQueryMain(args)
if self.debug:
print(result)
self.assertTrue("SELECT DISTINCT ?city ?cityLabel" in result)
[docs] def testCommandLineUsage(self):
'''
test the command line usage
'''
args=["-h"]
try:
stdout = io.StringIO()
with redirect_stdout(stdout):
queryMain(args)
self.fail("system exit expected")
except SystemExit:
pass
debug=self.debug
#debug=True
if debug:
print(stdout.getvalue())
self.assertTrue("--queryName" in stdout.getvalue())
[docs] def testQueryDocumentation(self):
'''
test QueryDocumentation
'''
show=self.debug
#show=True
queries=[
{
"endpoint":"https://query.wikidata.org/sparql",
"prefixes": [],
"lang": "sparql",
"name": "Nicknames",
"description": "https://stackoverflow.com/questions/70206791/sparql-i-have-individual-with-multiple-values-for-single-object-property-how",
"title": "Nick names of US Presidents",
"query":"""SELECT ?item ?itemLabel (GROUP_CONCAT(DISTINCT ?nickName; SEPARATOR=",") as ?nickNames)
WHERE
{
# president
?item wdt:P39 wd:Q11696.
?item wdt:P1449 ?nickName
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
} GROUP BY ?item ?itemLabel"""
},
{
"endpoint":"https://query.wikidata.org/sparql",
"prefixes": ["http://www.wikidata.org/entity/","http://commons.wikimedia.org/wiki/Special:FilePath/"],
"lang": "sparql",
"name": "CAS15",
"title": "15 Random substances with CAS number",
"description": "Wikidata SPARQL query showing the 15 random chemical substances with their CAS Number",
"query": """# List of 15 random chemical components with CAS-Number, formula and structure
# see also https://github.com/WolfgangFahl/pyLoDStorage/issues/46
# WF 2021-08-23
SELECT ?substance ?substanceLabel ?formula ?structure ?CAS
WHERE {
?substance wdt:P31 wd:Q11173.
?substance wdt:P231 ?CAS.
?substance wdt:P274 ?formula.
?substance wdt:P117 ?structure.
SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
}
LIMIT 15
"""
},
{
"endpoint":"https://query.wikidata.org/sparql",
"prefixes": ["http://www.wikidata.org/entity/"],
"lang": "sparql",
"name": "CityTop10",
"title": "Ten largest cities of the world",
"description": "Wikidata SPARQL query showing the 10 most populated cities of the world using the million city class Q1637706 for selection",
"query": """# Ten Largest cities of the world
# WF 2021-08-23
# see also http://wiki.bitplan.com/index.php/PyLoDStorage#Examples
# see also https://github.com/WolfgangFahl/pyLoDStorage/issues/46
SELECT DISTINCT ?city ?cityLabel ?population ?country ?countryLabel
WHERE {
VALUES ?cityClass { wd:Q1637706}.
?city wdt:P31 ?cityClass .
?city wdt:P1082 ?population .
?city wdt:P17 ?country .
SERVICE wikibase:label {
bd:serviceParam wikibase:language "en" .
}
}
ORDER BY DESC(?population)
LIMIT 10"""
},
{
"endpoint":"https://sophox.org/sparql",
"lang": "sparql",
"prefixes": [],
"query":
"""# count osm place type instances
# WF 2021-08-23
# see also http://wiki.bitplan.com/index.php/PyLoDStorage#Examples
# see also https://github.com/WolfgangFahl/pyLoDStorage/issues/46
SELECT (count(?instance) as ?count) ?placeType ?placeTypeLabel
WHERE {
VALUES ?placeType {
"city"
"town"
"village"
}
?instance osmt:place ?placeType
SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
}
GROUP BY ?placeType ?placeTypeLabel
ORDER BY ?count""",
"name": "OSM place types",
"title": "count OpenStreetMap place type instances",
"description":"""This SPARQL query
determines the number of instances available in the OpenStreetMap for the placeTypes city,town and village
"""}]
for queryMap in queries:
endpointUrl=queryMap.pop("endpoint")
endpoint=SPARQL(endpointUrl)
query=Query(**queryMap)
showYaml=False
if showYaml:
yamlMarkup=query.asYaml()
print(yamlMarkup)
try:
qlod=endpoint.queryAsListOfDicts(query.query)
for tablefmt in ["mediawiki","github","latex"]:
doc=query.documentQueryResult(qlod, tablefmt=tablefmt,floatfmt=".0f")
docstr=doc.asText()
if show:
print (docstr)
except Exception as ex:
print(f"{query.title} at {endpointUrl} failed: {ex}")
[docs] def testIssue61(self):
"""
tests different query path
see https://github.com/WolfgangFahl/pyLoDStorage/issues/61
"""
queriesPath=f"{os.path.dirname(__file__)}/../sampledata/scholia.yaml"
args=["-qp", f"{queriesPath}", "--list"]
stdout = io.StringIO()
with redirect_stdout(stdout):
queryMain(args, lang="sparql")
result = stdout.getvalue()
#debug=self.debug
debug=True
if debug:
print(result)
self.assertTrue("WorksAndAuthor" in result)
[docs]class TestEndpoints(Basetest):
"""
tests Endpoint
"""
[docs] def testEndpoints(self):
"""
tests getting and rawQuerying Endpoints
"""
debug=self.debug
#debug=True
endpoints=EndpointManager.getEndpoints()
qm=QueryManager(lang='sparql',debug=False)
query=qm.queriesByName["FirstTriple"]
self.assertTrue("wikidata" in endpoints)
for i,item in enumerate(endpoints.items()):
name,endpoint=item
if debug:
print (f"{i}:{name}")
resultFormat="json"
jsonStr=QueryMain.rawQuery(endpoint.endpoint, query.query, resultFormat, mimeType=None)
if debug:
print(jsonStr)
if __name__ == "__main__":
#import sys;sys.argv = ['', 'Test.testName']
unittest.main()