-
-
Notifications
You must be signed in to change notification settings - Fork 2
/
paper_thumbnail_mj.py
49 lines (42 loc) · 1.39 KB
/
paper_thumbnail_mj.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import requests, json, os
from datetime import datetime
from bs4 import BeautifulSoup
def get_image_urls(url):
urllist = []
headers = {'User-Agent': 'curl/7.84.0'}
page = requests.get(url, headers=headers, allow_redirects=True)
soup = BeautifulSoup(page.content, 'html.parser')
body = soup.find("body")
sc = list(body.find_all("script"))[-1].string
sc = str(sc)
print(soup)
script = soup.find('script', {'id': '__NEXT_DATA__'})
sc = script.contents[0]
print(sc)
# parse JSON
sc = json.loads(sc).get("props").get("pageProps").get("jobs")
for i in sc:
urllist.append(i.get("event").get("seedImageURL"))
return urllist
def download_image(urllist):
curdate = datetime.now().strftime("%Y%m%d")
os.makedirs(curdate, exist_ok=True)
os.chdir(curdate)
headers = {'User-Agent': 'curl/7.84.0'}
i = 1
for url in urllist:
page = requests.get(url, headers=headers, allow_redirects=True)
with open("apple-blog/public/assets/"+str(i) + ".png", 'wb') as f:
f.write(page.content)
i+=1
print(f"Downloaded {i}.png")
url = 'https://www.midjourney.com/showcase/recents'
url2 = 'https://www.midjourney.com/showcase/top'
urllist1 = get_image_urls(url)
urllist2 = get_image_urls(url2)
unique = set()
for i in urllist1:
unique.add(i)
for i in urllist2:
unique.add(i)
download_image(unique)