1. def get_proxy():
  2. return requests.get("http://127.0.0.1:5010/get/").text
  3. def delete_proxy(proxy):
  4. requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy))
  5. def downloadFile(isbn, url):
  6. headers = {'Proxy-Connection': 'keep-alive',
  7. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'}
  8. retry_count = 5
  9. proxy = get_proxy()
  10. is_push_queue = False
  11. while retry_count > 0:
  12. try:
  13. try:
  14. r = requests.get(url, timeout=2, proxies={"http": "http://{}".format(proxy)}, headers=headers,
  15. stream=True)
  16. except Exception as e:
  17. retry_count -= 1
  18. continue
  19. if r.status_code != 200:
  20. retry_count -= 1
  21. continue
  22. if r.status_code == 404:
  23. break
  24. try:
  25. book_name = r.headers['Content-Disposition'].split('filename=')[1].encode('ISO-8859-1').decode("utf8")
  26. book_name = eval(book_name)
  27. length = float(r.headers['content-length'])
  28. except Exception as e:
  29. retry_count -= 1
  30. continue
  31. path = "/root/sp_data/" + str(isbn) + "/"
  32. if not os.path.exists(path):
  33. os.makedirs(path)
  34. path = path + book_name
  35. f = open(path, 'wb')
  36. count = 0
  37. count_tmp = 0
  38. time1 = time.time()
  39. for chunk in r.iter_content(chunk_size=512):
  40. if chunk:
  41. f.write(chunk)
  42. count += len(chunk)
  43. if time.time() - time1 > 0.1:
  44. p = count / length * 100
  45. speed = (count - count_tmp) / 1024 / 1024 / 0.1
  46. count_tmp = count
  47. print(book_name + ': ' + formatFloat(p) + '%' + ' Speed: ' + formatFloat(speed) + 'M/S')
  48. time1 = time.time()
  49. f.close()
  50. assert length == os.path.getsize(path)
  51. return
  52. except Exception as e:
  53. print(e)
  54. retry_count -= 1
  55. is_push_queue = True
  56. if retry_count <= 0:
  57. dict_obj = {"isbn": isbn, "href": url}
  58. print("放回队列")
  59. redis_obj.rpush("isbn_list", json.dumps(dict_obj))
  60. delete_proxy(proxy)
  61. def formatFloat(num):
  62. return '{:.2f}'.format(num)