NLPCC 出版部分相关源码记录

目录

Download

Unzip

Author

Title

Affiliation

Check number of tex

Zip

Rename

Delete

---

Download

import requests
from bs4 import BeautifulSoup# 登录网站并获取登录后的 session
def login(username, password):login_url = 'https://example.com/login'session = requests.session()login_data = {'username': username,'password': password,# 其他登录参数}response = session.post(login_url, data=login_data)if response.status_code == 200:print("登录成功！")return sessionelse:print("登录失败！")return None# 获取文件列表页面中的文件链接
def get_file_links(session, file_list_url):response = session.get(file_list_url)soup = BeautifulSoup(response.text, 'html.parser')file_links = []# 使用 BeautifulSoup 解析文件列表页面，获取文件链接# 例如：file_links = soup.find_all('a', class_='file-link')return file_links# 批量下载文件
def download_files(session, file_links, download_path):for link in file_links:file_url = link['href']file_name = link.text.strip()response = session.get(file_url, stream=True)if response.status_code == 200:# 保存文件到本地with open(f"{download_path}/{file_name}", 'wb') as file:for chunk in response.iter_content(chunk_size=8192):file.write(chunk)print(f"{file_name} 下载成功！")else:print(f"{file_name} 下载失败！")def main():username = 'your_username'password = 'your_password'file_list_url = 'https://example.com/files'  # 文件列表页面的 URLdownload_path = 'downloaded_files'  # 本地下载路径# 登录网站并获取登录后的 sessionsession = login(username, password)if session:# 获取文件列表页面中的文件链接file_links = get_file_links(session, file_list_url)if file_links:# 批量下载文件download_files(session, file_links, download_path)else:print("未找到文件链接！")else:print("登录失败，请检查用户名和密码！")# if __name__ == "__main__":
#     main()

import requests
from bs4 import BeautifulSoupdef login(username, password):login_url = 'https://softconf.com/nlpcc/Main-2023/login/scmd.cgi?scmd=login'session = requests.session()login_data = {"username": username,"password": password}response = session.post(login_url, data=login_data)# print(response.text)if response.status_code == 200:print("登录成功！")return sessionelse:print("登录失败！")return None

username, passwd = "用户名", "密码"
session = login(username, passwd)

import reids = {214,215,220,221,222,225,229,233,235,238,239,241,246,250,251,252,254,256,258,260,264,271,285,292,299,301,306,307,308,}
file_list_url = "https://softconf.com/nlpcc/Main-2023/pub/scmd.cgi?scmd=manager&ToPage=monitorFinalSubmissions&FromPage=Main"
response = session.get(file_list_url)
soup = BeautifulSoup(response.text, 'html.parser')
table = soup.find('table', id='t1')
links = table.find_all('a')
all_urls = [link.get('href') for link in links]
urls = []
for i in range(len(all_urls)):if all_urls[i] and all_urls[i].startswith('scmd.cgi?scmd=submitPaperCustom'):if (m := re.search(r"passcode=(\d+)X-.+", all_urls[i])) is not None:# print(m.group(1))if int(m.group(1)) in ids:urls.append((int(m.group(1)), "https://softconf.com/nlpcc/Main-2023/pub/"+all_urls[i]))
print(len(urls)==len(ids))
print(urls)

import time
import os
from tqdm.auto import tqdmdef download_files(session, urls:dict, paper_id:int):for file_name, file_url in urls.items():response = session.get(file_url, stream=True)save_dir = f"./downloads/{paper_id}/"os.makedirs(save_dir, exist_ok=True)if response.status_code == 200:# 保存文件到本地with open(f"{save_dir}/{file_name}", 'wb') as file:for chunk in response.iter_content(chunk_size=8192):file.write(chunk)# print(f"{paper_id}_{file_name} 下载成功！")else:print(f"{paper_id}_{file_name} 下载失败！")for paper_id, url in tqdm(urls):response = session.get(url)soup = BeautifulSoup(response.text, 'html.parser')links = soup.find_all('a')urls_ = map(lambda link: link.get('href') if link else "", links)pdf_url = [link.get('href') for link in links if link.get('href') and link.get('href').endswith("fieldid=Final_Manuscript")][0]zip_url = [link.get('href') for link in links if link.get('href') and link.get('href').endswith("fieldid=Source_File")][0]copyright_url = [link.get('href') for link in links if link.get('href') and link.get('href').endswith("fieldid=CopyRight_Springer")][0]downloads_urls = {"Final_Manuscript.pdf": pdf_url, "Source_File.zip":zip_url, "CopyRight.pdf":copyright_url}downloads_urls = {"CopyRight.pdf":copyright_url}# print(downloads_urls)try:download_files(session, downloads_urls, paper_id)except:pass# breaktime.sleep(2)

Unzip

import zipfile
import os
import pathlibdef unzip_file(zip_filepath, dest_path):with zipfile.ZipFile(zip_filepath, 'r') as zip_ref:zip_ref.extractall(dest_path)# 使用方法
root_dir = pathlib.Path("./downloads/")
for directory in root_dir.iterdir():try:unzip_file(directory/"Source_File.zip", directory/"Source_File")except Exception as e:print(e)print(directory)# break

import pathlibroot_dir = pathlib.Path("./downloads/")
for directory in root_dir.iterdir():path = directory/"Source_File"path_true = pathlib.Path(path)dir_outputs_tex_true = path_true/"outputs_tex"dir_outputs_tex_true.mkdir(exist_ok=True)if (path/"submission.tex").exists():dir_outputs_tex = pathlib.PurePosixPath("outputs_tex")path_tex = pathlib.PurePosixPath("submission.tex")path_aux = dir_outputs_tex/"submission.aux"! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}! cd {path_true} & bibtex {path_aux}! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}else:print(directory)

def compile2pdf(directory):directory = pathlib.Path(directory)path = directory/"Source_File"path_true = pathlib.Path(path)dir_outputs_tex_true = path_true/"outputs_tex"dir_outputs_tex_true.mkdir(exist_ok=True)if (path/"submission.tex").exists():dir_outputs_tex = pathlib.PurePosixPath("outputs_tex")path_tex = pathlib.PurePosixPath("submission.tex")path_aux = dir_outputs_tex/"submission.aux"! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}! cd {path_true} & bibtex {path_aux}! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}! cd {path_true} & pdflatex -output-directory={dir_outputs_tex} -synctex=0 -interaction=nonstopmode -file-line-error {path_tex}else:print(directory)compile2pdf("downloads/306")

def is_same_file(file1, file2):with open(file1, 'rb') as f1, open(file2, 'rb') as f2:return f1.read() == f2.read()import PyPDF2from PyPDF2 import PdfReaderdef extract_text_from_pdf(file_path):with open(file_path, 'rb') as file:pdf = PdfReader(file)text = ""for page in range(len(pdf.pages)):text += pdf.pages[page].extract_text()return text, len(pdf.pages)def compare_pdfs(file_path1, file_path2):text1, n_1 = extract_text_from_pdf(file_path1)text2, n_2 = extract_text_from_pdf(file_path2)return text1 == text2, n_1, n_2root_dir = pathlib.Path("./downloads/")
for directory in root_dir.iterdir():camera_ready = directory/"Final_Manuscript.pdf"compiled = directory/"Source_File"/"outputs_tex"/"submission.pdf"try: ok, n1, n2 = compare_pdfs(camera_ready, compiled)if not ok:print(f"Not same: {directory}")print(n1, n2, sep='    ')except Exception as e:print(e)print(f"Fail to compare: {directory}")print("=========================================================================")

Author

import redef extract_author(tex_file_path):with open(tex_file_path, 'r', encoding='utf-8') as tex_file:tex_content = tex_file.read()# Use regular expression to find the \author partpattern = r"^\\author{\s*(.*?)\s*}\s+\%"matches = re.search(pattern, tex_content, re.DOTALL|re.MULTILINE)if matches:return matches.group(1)else:return ""tex_file_path = "downloads\\215\\Source_File\\submission.tex"  # Replace with the path to your .tex file
author = extract_author(tex_file_path)authors = []
root_dir = pathlib.Path("./downloads/")
for directory in root_dir.iterdir():tex_file_path = directory/"Source_File"/"submission.tex"print(f"------{directory}---------")if tex_file_path.exists():author = extract_author(tex_file_path)# author = re.sub(r"\\.*", "", author)# author = re.sub(r"[^\w\s]", "", author)# author = re.sub(r"\s*?\n\s*", ",", author)# author = author[:-1] if author.endswith(',') else author# author = re.sub(r'(?<=,)(?=[^,]*$)', 'and ', author)  #将最后一个逗号换成 `and`# # author = re.sub(r',(?=[^,]*$)', ' and ', author)  #将最后一个逗号换成 `and`authors.append(author)print(author)else:print(f"Fail to open tex: {tex_file_path}")authors.append("")print('====================================================================')

import pandas as pd# 将列表转换为DataFrame
df = pd.DataFrame(authors, columns=["author"])# 保存DataFrame到Excel文件
file_path = "./author.xlsx"
df.to_excel(file_path, index=False)

Title

import redef extract_title(tex_file_path):with open(tex_file_path, 'r', encoding='utf-8') as tex_file:tex_content = tex_file.read()# Use regular expression to find the \author partpattern = r"^\\title{\s*(.*?)\s*}\s+\%"matches = re.search(pattern, tex_content, re.DOTALL|re.MULTILINE)if matches:return matches.group(1)else:return ""# tex_file_path = "downloads\\215\\Source_File\\submission.tex"  # Replace with the path to your .tex file
# author = extract_author(tex_file_path)authors = []
root_dir = pathlib.Path("./downloads/")
for directory in root_dir.iterdir():tex_file_path = directory/"Source_File"/"submission.tex"print(f"------{directory}---------")if tex_file_path.exists():author = extract_title(tex_file_path)author = re.sub(r"\s*\\\\\s*", " ", author)author = re.sub(r"\\.*", "", author)authors.append(author)print(author)else:print(f"Fail to open tex: {tex_file_path}")authors.append("")print('====================================================================')

import pandas as pd# 将列表转换为DataFrame
df = pd.DataFrame(authors, columns=["title"])# 保存DataFrame到Excel文件
file_path = "./title.xlsx"
df.to_excel(file_path, index=False)

Affiliation

import redef extract_affiliation(tex_file_path):with open(tex_file_path, 'r', encoding='utf-8') as tex_file:tex_content = tex_file.read()# Use regular expression to find the \author partpattern = r"^\\institute{\s*(.*?)\s*}\s+\%"matches = re.search(pattern, tex_content, re.DOTALL|re.MULTILINE)if matches:return matches.group(1)else:return ""# tex_file_path = "downloads\\215\\Source_File\\submission.tex"  # Replace with the path to your .tex file
# author = extract_author(tex_file_path)authors = []
root_dir = pathlib.Path("./downloads/")
i = 2
for directory in root_dir.iterdir():tex_file_path = directory/"Source_File"/"submission.tex"print(f"------{i} {directory}---------")i += 1if tex_file_path.exists():author = extract_affiliation(tex_file_path)# author = re.sub(r"\s*\\\\\s*", " ", author)# author = re.sub(r"\\.*", "", author)authors.append(author)print(author)else:print(f"Fail to open tex: {tex_file_path}")authors.append("")print('====================================================================')

import pandas as pd# 将列表转换为DataFrame
df = pd.DataFrame(authors, columns=["affiliation"])# 保存DataFrame到Excel文件
file_path = "./affiliation.xlsx"
df.to_excel(file_path, index=False)

Check number of tex

import pathlib
root_dir = pathlib.Path("./downloads/")def num_tex(dirctory: pathlib.Path):num = 0for d in dirctory.iterdir():num += (d.suffix=='.tex')return numfor d in root_dir.iterdir():src = d/"Source_File"if num_tex(src)>1:print(d)

Zip

import os
import zipfiledef zip_directory(directory_path, zip_path):"""压缩目录到zip文件:param directory_path: 要压缩的目录路径:param zip_path: zip文件保存路径"""with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:for root, _, files in os.walk(directory_path):for file in files:file_path = os.path.join(root, file)zipf.write(file_path, os.path.relpath(file_path, directory_path))# # 示例用法
# directory_to_compress = '/path/to/source_directory'
# zip_file_path = '/path/to/destination.zip'
# zip_directory(directory_to_compress, zip_file_path)

Rename

import pathlib
root_dir = pathlib.Path("./downloads/")for d in list(root_dir.iterdir()):src = d/"Source_File"zip_directory(src, src.parent/"source.zip")submi = d/"Final_Manuscript.pdf"submi.rename(submi.with_name("submission.pdf"))cprt = d/"CopyRight.pdf"cprt.rename(cprt.rename(cprt.with_name("copyright.pdf")))

Delete

import pathlib
import shutil
import os
root_dir = pathlib.Path("./downloads/")for d in list(root_dir.iterdir()):src = d/"Source_File.zip"os.remove(src)