From e03feaac89ead4dc136f79483f80cad0f72324d6 Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 7 Sep 2014 12:17:47 +0200 Subject: [PATCH 01/10] Fix url scraping in getStreamUrlNew() Seems a site update has changed from using double to single quotes which breaks the url scraper. Updated regex to catch both single and double quotes. --- default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.py b/default.py index 54667de..473d937 100644 --- a/default.py +++ b/default.py @@ -215,7 +215,7 @@ def playVideoNew(url): def getStreamUrlNew(url): content = getUrl(url) - match = re.compile('arte_vp_url="(.+?)"', re.DOTALL).findall(content) + match = re.compile('arte_vp_url=[\'"](.+?)[\'"]', re.DOTALL).findall(content) if "concert.arte.tv" in url: url = match[0] content = getUrl(url) From 21dcbbde99b976dbff35378683d67040b7d70d7b Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Fri, 24 Oct 2014 22:40:48 +0200 Subject: [PATCH 02/10] Fix URL for new episodes list. --- default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.py b/default.py index 473d937..3bb7689 100644 --- a/default.py +++ b/default.py @@ -39,7 +39,7 @@ def index(): for filter in regionFilters: regionFilter += filter.replace("'","").strip()+"%2C" regionFilter = regionFilter[:-3] - addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7/plus_recentes.json?regions="+regionFilter, "listVideosNew", "") + addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7.json", "listVideosNew", "") addDir(translation(30002), baseUrl+"/guide/"+language+"/plus7/selection.json?regions="+regionFilter, "listVideosNew", "") addDir(translation(30003), baseUrl+"/guide/"+language+"/plus7/plus_vues.json?regions="+regionFilter, "listVideosNew", "") addDir(translation(30004), baseUrl+"/guide/"+language+"/plus7/derniere_chance.json?regions="+regionFilter, "listVideosNew", "") From 3cf48e8894e5ce6979053eea0ba2d79db6eb811c Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sat, 25 Oct 2014 18:31:37 +0200 Subject: [PATCH 03/10] Cleanup listVideosNew - convert items to utf-8 encoded strings using helper function str_item(). --- default.py | 69 ++++++++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/default.py b/default.py index 3bb7689..78708b0 100644 --- a/default.py +++ b/default.py @@ -53,50 +53,43 @@ def index(): xbmcplugin.endOfDirectory(pluginhandle) + +def str_item(item): + '''Convert item to utf-8 encoded string object''' + if not item: + item = '' + if not isinstance(item, basestring): + try: + item = str(item) + except: + item = '' + return item.encode('utf-8', 'replace') + + def listVideosNew(url): xbmcplugin.setContent(pluginhandle, "episodes") content = getUrl(url) content = json.loads(content) for item in content["videos"]: - title = item["title"].encode('utf-8') - try: - desc = item["desc"].encode('utf-8') - except: - desc = "" - try: - duration = str(item["duration"]) - except: - duration = "" - try: - date = item["airdate_long"].encode('utf-8') - except: - date = "" - try: - url = item["url"] - except: - url = "" - try: - thumb = item["image_url"] - except: - thumb = "" - try: - channels = item["video_channels"].encode('utf-8') - except: - channels = "" - try: - views = str(item["video_views"]) - except: - views = "" - try: - until = item["video_rights_until"].encode('utf-8') - except: - until = "" - try: - rank = str(item["video_rank"]) - except: - rank = "" + title = str_item(item.get("title", "No Title")) + desc = str_item(item.get("desc", "")) + duration = str_item(item.get("duration", "")) + date = str_item(item.get("airdate_long", "")) + url = str_item(item.get("url", "")) + thumb = str_item(item.get("image_url", "")) + channels = str_item(item.get("video_channels", "")) + views = str_item(item.get("video_views", "")) + until = str_item(item.get("video_rights_until", "")) + rank = str_item(item.get("video_rank", "")) + desc = views+" | "+date+"\n"+channels+"\n"+desc - addLink(cleanTitle(title), baseUrl+url, 'playVideoNew', thumb, desc, duration) + addLink(cleanTitle(title), + baseUrl+url, + 'playVideoNew', + thumb, + desc, + duration, + ) xbmcplugin.endOfDirectory(pluginhandle) if forceViewMode: xbmc.executebuiltin('Container.SetViewMode('+viewMode+')') From 1001fcddedea42191e21b5e941fa249a430fbab7 Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sat, 25 Oct 2014 18:37:58 +0200 Subject: [PATCH 04/10] Remove calls to cleanTitle() - Doesn't seem to be required as the title data from json is clean from html encodings - Keep the function for now in case needed later. --- default.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/default.py b/default.py index 78708b0..3be8e52 100644 --- a/default.py +++ b/default.py @@ -83,7 +83,7 @@ def listVideosNew(url): rank = str_item(item.get("video_rank", "")) desc = views+" | "+date+"\n"+channels+"\n"+desc - addLink(cleanTitle(title), + addLink(title, baseUrl+url, 'playVideoNew', thumb, @@ -103,11 +103,11 @@ def listSearchVideos(urlMain): for i in range(1, len(spl), 1): entry = spl[i] match = re.compile('alt="(.+?)"', re.DOTALL).findall(entry) - title = cleanTitle(match[0]) + title = match[0] match = re.compile('data-description="(.+?)"', re.DOTALL).findall(entry) desc = "" if match: - desc = cleanTitle(match[0]) + desc = match[0] match = re.compile('

.+?.+?(.+?)

.+?

.+?.+?(.+?)

', re.DOTALL).findall(entry) if match: date = match[0][0].strip() @@ -135,7 +135,7 @@ def listCats(type, regionFilter): content = content[:content.find('')] match = re.compile('(.+?)', re.DOTALL).findall(content) for url, title in match: - title = cleanTitle(title) + title = title url = baseUrl+url.replace("?", ".json?").replace("&", "&")+"®ions="+regionFilter addDir(title, url, 'listVideosNew', "") xbmcplugin.endOfDirectory(pluginhandle) @@ -187,7 +187,7 @@ def listConcerts(url=""): for i in range(1, len(spl), 1): entry = spl[i] match = re.compile('title="(.+?)"', re.DOTALL).findall(entry) - title = cleanTitle(match[0]) + title = match[0] match = re.compile('href="(.+?)"', re.DOTALL).findall(entry) url = baseUrlConcert+match[0] match = re.compile('src="(.+?)"', re.DOTALL).findall(entry) From 8937cdae1c2126227c625c02073beca9cd21d28e Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Oct 2014 12:32:37 +0100 Subject: [PATCH 05/10] Remove unused os module import; PEP 8 formating --- default.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/default.py b/default.py index 3be8e52..4311533 100644 --- a/default.py +++ b/default.py @@ -5,14 +5,13 @@ import socket import sys import re -import os import json import xbmcplugin import xbmcaddon import xbmcgui -#addon = xbmcaddon.Addon() -#addonID = addon.getAddonInfo('id') +# addon = xbmcaddon.Addon() +# addonID = addon.getAddonInfo('id') addonID = "plugin.video.arte_tv" addon = xbmcaddon.Addon(id=addonID) socket.setdefaulttimeout(30) From ee515e5548cef111fbc4a3cc0b65b97261581018 Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Oct 2014 12:33:48 +0100 Subject: [PATCH 06/10] Cleanup index() a bit; PEP 8 formating. --- default.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/default.py b/default.py index 4311533..f3bc5f0 100644 --- a/default.py +++ b/default.py @@ -29,32 +29,49 @@ streamingType = addon.getSetting("streamingType") streamingType = ["HTTP", "RTMP"][int(streamingType)] +guide = '%s/guide/%s' % (baseUrl, language) +new_json = 'plus7.json' +selection_json = 'plus7/selection.json' +lastchance_json = 'plus7/derniere_chance.json' +mostviewed_json = 'plus7/plus_vues.json' + + +def jsonUrl(jsonfile, regions=''): + url = '%s/%s' % (guide, jsonfile) + if regions: + url = '%s?regions=%s' % (url, regions) + return url + def index(): content = getUrl(baseUrl+"/artews/js/geolocation.js") - match = re.compile('arte_geoip_zone_codes.+?return new Array\\((.+?)\\)', re.DOTALL).findall(content) + match = re.compile('arte_geoip_zone_codes.+?return new Array\\((.+?)\\)', + re.DOTALL).findall(content) regionFilters = match[0].split(",") regionFilter = "" for filter in regionFilters: - regionFilter += filter.replace("'","").strip()+"%2C" + regionFilter += filter.replace("'", "").strip()+"%2C" regionFilter = regionFilter[:-3] - addDir(translation(30001), baseUrl+"/guide/"+language+"/plus7.json", "listVideosNew", "") - addDir(translation(30002), baseUrl+"/guide/"+language+"/plus7/selection.json?regions="+regionFilter, "listVideosNew", "") - addDir(translation(30003), baseUrl+"/guide/"+language+"/plus7/plus_vues.json?regions="+regionFilter, "listVideosNew", "") - addDir(translation(30004), baseUrl+"/guide/"+language+"/plus7/derniere_chance.json?regions="+regionFilter, "listVideosNew", "") + addDir(translation(30001), jsonUrl(new_json), + "listVideosNew", "") + addDir(translation(30002), jsonUrl(selection_json, regionFilter), + "listVideosNew", "") + addDir(translation(30003), jsonUrl(lastchance_json, regionFilter), + "listVideosNew", "") + addDir(translation(30004), jsonUrl(mostviewed_json, regionFilter), + "listVideosNew", "") addDir(translation(30005), "by_channel", "listCats", "", regionFilter) addDir(translation(30006), "by_cluster", "listCats", "", regionFilter) addDir(translation(30007), "by_date", "listCats", "", regionFilter) addDir(translation(30008), "", "search", "") addDir(translation(30012), "", "listConcertsMain", "") - if regionFilter!="ALL": + if regionFilter != "ALL": addLink(translation(30009), "", "playLiveStream", icon) xbmcplugin.endOfDirectory(pluginhandle) - def str_item(item): - '''Convert item to utf-8 encoded string object''' + '''Convert item to utf-8 encoded string object''' if not item: item = '' if not isinstance(item, basestring): From 54b93d82b1f68377de08ba32c692562c2f59bbd8 Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Apr 2015 12:34:33 +0200 Subject: [PATCH 07/10] Release 2.1.1 --- addon.xml | 2 +- changelog.txt | 4 ++++ default.py | 45 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/addon.xml b/addon.xml index fccd4f1..fd2f30d 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + diff --git a/changelog.txt b/changelog.txt index de60c5f..25a30cb 100644 --- a/changelog.txt +++ b/changelog.txt @@ -22,3 +22,7 @@ - Fixed site changes (concerts) 2.0.9 - Fixed geoblocking filter +2.1.0 +- Fixed concert livestreams +2.1.1 +- Added support for concert collections diff --git a/default.py b/default.py index 54667de..100ce62 100644 --- a/default.py +++ b/default.py @@ -20,6 +20,8 @@ forceViewMode = addon.getSetting("forceView") == "true" useThumbAsFanart = addon.getSetting("useThumbAsFanart") == "true" viewMode = str(addon.getSetting("viewIDNew")) +addonDir = xbmc.translatePath(addon.getAddonInfo('path')) +defaultFanart = os.path.join(addonDir ,'fanart.png') icon = xbmc.translatePath('special://home/addons/'+addonID+'/icon.png') baseUrl = "http://www.arte.tv" baseUrlConcert = "http://concert.arte.tv" @@ -164,7 +166,7 @@ def search(): def listConcertsMain(): addDir(translation(30002), "", "listConcerts", "") - addDir(translation(30003), baseUrlConcert+"/"+language+"/videos/all?sort=mostviewed", "listConcerts", "") + addDir("Collections", "", "listCollections", "") addDir(translation(30011), baseUrlConcert+"/"+language+"/videos/all", "listConcerts", "") addDir(translation(30013), baseUrlConcert+"/"+language+"/videos/rockpop", "listConcerts", "") if language=="de": @@ -199,7 +201,10 @@ def listConcerts(url=""): url = baseUrlConcert+match[0] match = re.compile('src="(.+?)"', re.DOTALL).findall(entry) thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/") - addLink(title, url, 'playVideoNew', thumb, "") + if "node-eventp" in entry: + addDir(title, url, "listConcerts", thumb) + elif "node-videop" in entry: + addLink(title, url, 'playVideoNew', thumb, "") match = re.compile('
  • .+?href="(.+?)"', re.DOTALL).findall(content) if match: addDir(translation(30010), baseUrlConcert+match[0], "listConcerts", "") @@ -208,6 +213,29 @@ def listConcerts(url=""): xbmc.executebuiltin('Container.SetViewMode('+viewMode+')') +def listCollections(): + content = getUrl("http://concert.arte.tv/"+language+"/collections.xml") + spl = content.split('') + for i in range(1, len(spl), 1): + entry = spl[i] + match = re.compile('(.+?)', re.DOTALL).findall(entry) + title = cleanTitle(match[0]) + match = re.compile('field-name-eventp-videos-count.*?>(.+?)<', re.DOTALL).findall(entry) + if match: + count = match[0].strip() + if language=="de": + count = count.replace("vidéos","Videos") + title += " ("+count+")" + match = re.compile('(.+?)', re.DOTALL).findall(entry) + url = match[0] + match = re.compile('data-src="(.+?)"', re.DOTALL).findall(entry) + thumb = match[0].replace("/alw_rectangle_376/","/alw_rectangle_690/").replace("/alw_highlight_480/","/alw_rectangle_690/") + addDir(title, url, "listConcerts", thumb) + xbmcplugin.endOfDirectory(pluginhandle) + if forceViewMode: + xbmc.executebuiltin('Container.SetViewMode('+viewMode+')') + + def playVideoNew(url): listitem = xbmcgui.ListItem(path=getStreamUrlNew(url)) xbmcplugin.setResolvedUrl(pluginhandle, True, listitem) @@ -215,16 +243,19 @@ def playVideoNew(url): def getStreamUrlNew(url): content = getUrl(url) - match = re.compile('arte_vp_url="(.+?)"', re.DOTALL).findall(content) + match = re.compile("arte_vp_url='(.+?)'", re.DOTALL).findall(content) if "concert.arte.tv" in url: url = match[0] content = getUrl(url) match1 = re.compile('"HTTP_SQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content) match2 = re.compile('"HTTP_EQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content) + match3 = re.compile('"RMTP_HQ":.*?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content) if match1 and maxVideoQuality == "720p": return match1[0].replace("\\","") elif match2: return match2[0].replace("\\","") + elif match3: + return match3[0][0].replace("\\","")+match3[0][1].replace("\\","")+" swfUrl=http://www.arte.tv/flash/mediaplayer/mediaplayer.swf live=1 swfVfy=1" elif streamingType=="HTTP": url = match[0].replace("/player/","/") content = getUrl(url) @@ -306,6 +337,8 @@ def addLink(name, url, mode, iconimage, desc="", duration=""): liz.setProperty('IsPlayable', 'true') if useThumbAsFanart and iconimage!=icon: liz.setProperty("fanart_image", iconimage) + else: + liz.setProperty("fanart_image", defaultFanart) liz.addContextMenuItems([(translation(30020), 'RunPlugin(plugin://'+addonID+'/?mode=queueVideo&url='+urllib.quote_plus(u)+'&name='+urllib.quote_plus(name)+')',)]) ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz) return ok @@ -316,6 +349,10 @@ def addDir(name, url, mode, iconimage, regionFilter=""): ok = True liz = xbmcgui.ListItem(name, iconImage=icon, thumbnailImage=iconimage) liz.setInfo(type="Video", infoLabels={"Title": name}) + if useThumbAsFanart and iconimage and iconimage!=icon: + liz.setProperty("fanart_image", iconimage) + else: + liz.setProperty("fanart_image", defaultFanart) ok = xbmcplugin.addDirectoryItem(handle=int(sys.argv[1]), url=u, listitem=liz, isFolder=True) return ok @@ -341,6 +378,8 @@ def addDir(name, url, mode, iconimage, regionFilter=""): playLiveStream() elif mode == 'listConcerts': listConcerts(url) +elif mode == 'listCollections': + listCollections() elif mode == 'listConcertsMain': listConcertsMain() elif mode == 'search': From 1e790029e78b568e210e03e07a4c82c78e12cdae Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Apr 2015 15:23:05 +0200 Subject: [PATCH 08/10] import os --- default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/default.py b/default.py index 609a220..6a7d512 100644 --- a/default.py +++ b/default.py @@ -1,5 +1,6 @@ #!/usr/bin/python # -*- coding: utf-8 -*- +import os import urllib import urllib2 import socket From fcf7add92fce7f034cdd181cfe345c43e9272dec Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Apr 2015 15:23:31 +0200 Subject: [PATCH 09/10] url is not full URL We don't need to prepend baseUrl. --- default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.py b/default.py index 6a7d512..ed3848b 100644 --- a/default.py +++ b/default.py @@ -103,7 +103,7 @@ def listVideosNew(url): desc = views+" | "+date+"\n"+channels+"\n"+desc addLink(title, - baseUrl+url, + url, 'playVideoNew', thumb, desc, From 07ee17de47ad5237b39869459b667f21e1efbd53 Mon Sep 17 00:00:00 2001 From: Scott Bahling Date: Sun, 26 Apr 2015 18:14:42 +0200 Subject: [PATCH 10/10] Make use of json structures with getStreamUrlNew() Instead of regex we use the json module to find the stream url. --- default.py | 87 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 34 deletions(-) diff --git a/default.py b/default.py index ed3848b..d38bbd6 100644 --- a/default.py +++ b/default.py @@ -251,42 +251,61 @@ def playVideoNew(url): xbmcplugin.setResolvedUrl(pluginhandle, True, listitem) +def getBestStream(VSR): + '''Return the best quality stream from an arte VSR json structure. + Try preferred streamingType first and fall back to other type. + ''' + + vid_format_tmpl = u'{streamtype}_{qual}_{lang}' + lang_code = {'de': '1', 'fr': '2'} + + if maxVideoQuality == '720p': + quals = ['SQ', 'HQ', 'EQ', 'MQ'] + else: + quals = ['EQ', 'MQ'] + + # Videos use HTTP_MP4 concerts use HTTP. We always try both. + streamtypes = {'RTMP': ['RMTP', 'HTTP_MP4', 'HTTP'], + 'HTTP': ['HTTP_MP4', 'HTTP', 'RMTP'], + } + + lang = lang_code.get(language.lower(), 'de') + for streamtype in streamtypes[streamingType]: + streamUrl = None + for qual in quals: + vid_format = vid_format_tmpl.format(streamtype=streamtype, + qual=qual, + lang=lang, + ) + try: + streamUrl = VSR[vid_format]['url'] + streamer = VSR[vid_format].get('streamer', '') + if streamer: + streamUrl = '%s/%s' % (streamer, streamUrl) + break + except: + pass + + if streamUrl is not None: + break + + return streamUrl + + def getStreamUrlNew(url): content = getUrl(url) - match = re.compile('arte_vp_url=[\'"](.+?)[\'"]', re.DOTALL).findall(content) - if "concert.arte.tv" in url: - url = match[0] - content = getUrl(url) - match1 = re.compile('"HTTP_SQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content) - match2 = re.compile('"HTTP_EQ_1":.+?"url":"(.+?)"', re.DOTALL).findall(content) - match3 = re.compile('"RMTP_HQ":.*?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content) - if match1 and maxVideoQuality == "720p": - return match1[0].replace("\\","") - elif match2: - return match2[0].replace("\\","") - elif match3: - return match3[0][0].replace("\\","")+match3[0][1].replace("\\","")+" swfUrl=http://www.arte.tv/flash/mediaplayer/mediaplayer.swf live=1 swfVfy=1" - elif streamingType=="HTTP": - url = match[0].replace("/player/","/") - content = getUrl(url) - match1 = re.compile('"HBBTV","VQU":"SQ","VMT":"mp4","VUR":"(.+?)"', re.DOTALL).findall(content) - match2 = re.compile('"HBBTV","VQU":"EQ","VMT":"mp4","VUR":"(.+?)"', re.DOTALL).findall(content) - if match1 and maxVideoQuality == "720p": - return match1[0] - elif match2: - return match2[0] - elif streamingType=="RTMP": - url = match[0] - content = getUrl(url) - match1 = re.compile('"RTMP_SQ_1":.+?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content) - match2 = re.compile('"RTMP_MQ_1":.+?"streamer":"(.+?)","url":"(.+?)"', re.DOTALL).findall(content) - if match1 and maxVideoQuality == "720p": - base = match1[0][0] - playpath = match1[0][1] - elif match2: - base = match2[0][0] - playpath = match2[0][1] - return base+" playpath=mp4:"+playpath + vp_url_re = 'arte_vp_url=[\'"](.+?)[\'"]' + match = re.compile(vp_url_re, re.DOTALL).findall(content) + + url = match[0] + content = getUrl(url) + content = json.loads(content) + + VSR = content['videoJsonPlayer']['VSR'] + + url = getBestStream(VSR) + + return url def queueVideo(url, name):