Data Scraping Example: MIT Courses

library(rvest)

url = "https://ocw.mit.edu/courses/most-visited-courses/"
webpage = read_html(url, encoding = "UTF-8")
course_nodes <- html_nodes(webpage,xpath = '//tr[@class="odd" or @class="even"]')
print(length(course_nodes))

## [1] 20

for (course in course_nodes){
  info <- html_nodes(course, xpath = './/td/a')
  courselink <- html_attr(info[1], "href")
  print(html_text(info[1]))
  print(html_text(info[2]))
  print(html_text(info[3]))
  print(paste0("https://ocw.mit.edu", courselink))
  writeLines("") 
}

## [1] "6.0001"
## [1] "Introduction to Computer Science and Programming in Python"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-0001-introduction-to-computer-science-and-programming-in-python-fall-2016"
## 
## [1] "18.01SC"
## [1] "Single Variable Calculus"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-01sc-single-variable-calculus-fall-2010"
## 
## [1] "6.006"
## [1] "Introduction to Algorithms"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-006-introduction-to-algorithms-fall-2011"
## 
## [1] "15.S12"
## [1] "Blockchain and Money"
## [1] "Graduate"
## [1] "https://ocw.mit.edu/courses/sloan-school-of-management/15-s12-blockchain-and-money-fall-2018"
## 
## [1] "18.06"
## [1] "Linear Algebra"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-06-linear-algebra-spring-2010"
## 
## [1] "18.02SC"
## [1] "Multivariable Calculus"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-02sc-multivariable-calculus-fall-2010"
## 
## [1] "18.06SC"
## [1] "Linear Algebra"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-06sc-linear-algebra-fall-2011"
## 
## [1] "8.01SC"
## [1] "Classical Mechanics"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/physics/8-01sc-classical-mechanics-fall-2016"
## 
## [1] "18.01"
## [1] "Single Variable Calculus"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-01-single-variable-calculus-fall-2006"
## 
## [1] "18.05"
## [1] "Introduction to Probability and Statistics"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-05-introduction-to-probability-and-statistics-spring-2014"
## 
## [1] "6.00SC"
## [1] "Introduction to Computer Science and Programming"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-00sc-introduction-to-computer-science-and-programming-spring-2011"
## 
## [1] "6.042J"
## [1] "Mathematics for Computer Science"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-042j-mathematics-for-computer-science-spring-2015"
## 
## [1] "6.042J"
## [1] "Mathematics for Computer Science"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-042j-mathematics-for-computer-science-fall-2010"
## 
## [1] "18.03SC"
## [1] "Differential Equations"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-03sc-differential-equations-fall-2011"
## 
## [1] "18.S096"
## [1] "Topics in Mathematics with Applications in Finance"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/mathematics/18-s096-topics-in-mathematics-with-applications-in-finance-fall-2013"
## 
## [1] "6.0002"
## [1] "Introduction to Computational Thinking and Data Science"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-0002-introduction-to-computational-thinking-and-data-science-fall-2016"
## 
## [1] "8.04"
## [1] "Quantum Physics I"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/physics/8-04-quantum-physics-i-spring-2016"
## 
## [1] "6.034"
## [1] "Artificial Intelligence"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-034-artificial-intelligence-fall-2010"
## 
## [1] "14.01"
## [1] "Principles of Microeconomics"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/economics/14-01-principles-of-microeconomics-fall-2018"
## 
## [1] "14.01SC"
## [1] "Principles of Microeconomics"
## [1] "Undergraduate"
## [1] "https://ocw.mit.edu/courses/economics/14-01sc-principles-of-microeconomics-fall-2011"