programme-specification-scraper/programme-scraper.py at master · SaileshPatel/programme-specification-scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import tabula
import requests
from bs4 import BeautifulSoup

def extract_course_info(course):
    path = spec_to_get + ".pdf"
    filename = 'pdf_' + spec_to_get

    try:
        pdf_text = tabula.read_pdf(path, lattice=True, pandas_options={'header': None}, pages="all", multiple_tables=True)

        specification_data = {}

        for index, page in enumerate(pdf_text):
            # page = page in PDF file
            # index = page no i.e 0 = 1, 1 = 2, etc
            if(index == 0):
                specification_data[page.iloc[0, 0]] = page.iloc[0, 1] # Programe Title
                specification_data['School of Study'] = page.iloc[2, 1]
                specification_data['Degree Type'] = page.iloc[3, 1]
                specification_data['Location'] = page.iloc[9, 1]
                specification_data['Programme Length'] = page.iloc[10, 1]
                specification_data['Total Credits'] = page.iloc[11, 1]
                specification_data['Accredited'] = page.iloc[12, 1]


                for index, value in page.iloc[6].items():
                    if value == 'X':
                        if index == 1:
                            specification_data['Attendance Type'] = "Full Time"
                        elif index == 2:
                            specification_data['Attendance Type'] = "Part Time"

                for index, value in page.iloc[8].items():
                    if value == 'X':
                        if index == 1:
                            specification_data['Delivery Pattern'] = "Campus-based"
                        elif index == 2:
                            specification_data['Delivery Pattern'] = "Work-based"
                        elif index == 3:
                            specification_data['Delivery Pattern'] = "Online/distance"


        print(specification_data)
    except Exception as ex:
        print("Your PDF cannot be scraped.")
        print(ex)

def web_scraping(school, course):
    try:
        res = requests.get(course)
        soup = BeautifulSoup(res.content, 'lxml')
        course_info = {}
        database_info = {}
        UCAS_code = ""

        if school == 'ABS' :
            return None
        elif school == 'AMS' :
            return None
        elif school == 'EAS' :
            #['Career Prospects', 'Entry Requirements & Fees for 2020', 'Course Outline & Modules', 'Placement Year', 'Learning, Teaching & Assessment', 'Teaching Staff', 'Contact Us']
            for info_accordion in soup.find_all("div", {"class": "accordion"}):
                item_name = "" # temporary storage for name of the key
                for title in info_accordion.find_all("a", {"class": "accordion__title"}):
                    item_name = title.text
                    course_info[item_name] = ""
                for info in info_accordion.find_all("div", {"class": "accordion__inner"}):
                    if item_name == "Entry Requirements & Fees for 2020":
                        for p in info_accordion.find_all("p", {"class": None}):
                            for title_text in p.findAll("strong"):
                                #print(title_text)
                                if title_text.text == "UCAS Code:":
                                    UCAS_code = p.find_all(text=True, recursive=False)[0].string.strip()

                    course_info[item_name] = info.text.strip().replace('\n', ' ').encode("ascii", "ignore")

            course_info['UCAS Code'] = UCAS_code
        elif school == 'LHS' :
            return None
        elif school == 'LSS':
            return None
        else:
            print("invalid school choice")
        print(course_info)
    except Exception as ex:
        print("The URL you have provided cannot be scraped.")
        print(ex)

if __name__ == '__main__':
    scraping_method = input("Type in P for PDF scraper, W for Web Scraping, or Q to Quit: ")
    if scraping_method == 'W':
        # https://www2.aston.ac.uk/study/courses/computer-science-bsc
        course_school = input("Enter the school which the course you want to scrape belongs to:\nABS - Aston Business School\nAMS - Aston Medical School\nEAS - Engineering & Applied Sciences\nLHS - Life & Health Sciences\nLSS - Languages & Social Sciences\n")
        web_spec_to_get = input("Enter the website from which you want the course specification from: ")
        web_scraping(course_school, web_spec_to_get)
    elif scraping_method == 'P':
        spec_to_get = input("Enter in the course specification you would like to scrape: ")
        # ProgrammeSpecifications/BScComputerScience
        # BScDigitalDegreeApprenticeship
        extract_course_info(spec_to_get)
    else:
        exit()