learning from 100 Days of Code: The Complete Python Pro Bootcamp for 2022
00. Get datas
import pandas as pd
df = pd.read_csv('salaries_by_college_major.csv')
01. Basic DataFrame INFO
.head() / .tail() / .shape / .columns
02. Look for NaN & clean up
.findna() / .dropna()
03. access entire columns
df['column name']
df[['column name 1', 'column name 2', 'column name 3']]
04. access individual cells
df['column name'][index]
df['column name'].loc[index]
05. largest and smallest values
.max() / .min()
.idxmax() / .idxmin()
06. sort the DataFrame
.sort_values('column', ascending=False)
07. add new columns
.insert(location, column_name, column)
08. create category
.groupby('category_column').count()
09. format the datas
pd.options.display.float_format = '{:,.2f}'.format
Scratching datas by BeautifulSoup
import requests
from bs4 import BeautifulSoup
import time
url = "https://www.payscale.com/college-salary-report/majors-that-pay-you-back/bachelors"
# write the first row in csv
with open("salaries_by_college_major_new.csv", 'w') as file_create:
file_create.write("Major,Early Career Pay,Mid-Career Pay,High Meaning\n")
def record_data(major_list, salary_list):
for major_index in range(0, len(major_list)):
salary_index = major_index * 3
with open("salaries_by_college_major_new.csv", 'a') as file:
file.write(f"{major_list[major_index]},"
f"{float(salary_list[salary_index].replace(',', '').replace('$', ''))},"
f"{float(salary_list[salary_index + 1].replace(',', '').replace('$', ''))},"
f"{salary_list[salary_index + 2]}\n")
for page in range(1, 35): # 35
web_data = requests.get(f"{url}/page/{page}").text
soup = BeautifulSoup(web_data, "html.parser")
majors_html = soup.select(".csr-col--school-name .data-table__value")
majors = [major.text for major in majors_html]
salaries_html = soup.select(".csr-col--right .data-table__value")
salaries = [salary.text for salary in salaries_html]
record_data(majors, salaries)
time.sleep(5)
