Skip to content

Remove the language filter for wikidata labels. #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gmodena wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Remove the language filter for wikidata labels. #26

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
237410e
Remove the language filter for wikidata labels.
gmodena May 5, 2021
61d156c
Fix. specify memory unit of measurement
gmodena May 6, 2021
894d5c2
WIP: memory tweaks to generate enwiki.
gmodena May 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions algorithm.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,23 @@
"# Under the hood the library uses findspark to initialise\n",
"# Spark's environment. pyspark imports will be available \n",
"# after initialisation\n",
"spark = get_session(type='regular', app_name=\"ImageRec-DEV Training\")\n",
"# spark = get_session(type='regular', extra_settings={'spark.driver.maxResultSize': '2048M'})\n",
"import os\n",
"import findspark\n",
"SPARK_HOME = os.environ.get(\"SPARK_HOME\", \"/usr/lib/spark2\")\n",
"findspark.init(SPARK_HOME)\n",
"findspark._add_to_submit_args('--driver-memory 64G --conf \"spark.driver.extraJavaOptions=-XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps\"')\n",
"master = 'yarn'\n",
"app_name = 'gmodena-imagematching-driver-highmem'\n",
"import pyspark\n",
"import pyspark.sql"
"from pyspark.sql import SparkSession\n",
"\n",
"spark = (\n",
" SparkSession.builder\n",
" .master(master)\n",
" .config('spark.driver.maxResultSize', '2048M')\n",
" .appName(app_name)\n",
" ).getOrCreate()"
]
},
{
Expand Down Expand Up @@ -301,7 +315,6 @@
" LATERAL VIEW explode(labels) t AS label_lang,label_val\n",
" LATERAL VIEW OUTER explode(claims) c AS claim\n",
" WHERE typ='item'\n",
" AND t.label_lang='\"\"\"+label_lang+\"\"\"'\n",
" AND snapshot='\"\"\"+snapshot+\"\"\"'\n",
" AND claim.mainSnak.property in ('P18','P31','P373')\n",
" GROUP BY id,label_val\n",
Expand Down Expand Up @@ -597,10 +610,10 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.1"
"version": "3.7.3"
},
"toc-showtags": true
},
"nbformat": 4,
"nbformat_minor": 4
}
}