Automatically migrated from Gitolite
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

43 lines
1.0 KiB

9 years ago
9 years ago
9 years ago
  1. #!/usr/bin/python
  2. import webshotslib, time
  3. to_parse = []
  4. to_parse_count = 0
  5. users = []
  6. users_count = 0
  7. users_last_save = 0
  8. print "Starting...",
  9. for category in webshotslib.get_category_listings("http://community.webshots.com/"):
  10. to_parse.append(category)
  11. to_parse_count += 1
  12. print "\rTotal pages to be parsed: %d" % to_parse_count,
  13. for listing_page in webshotslib.get_user_listings(category):
  14. to_parse.append(listing_page)
  15. to_parse_count += 1
  16. print "\rTotal pages to be parsed: %d" % to_parse_count,
  17. time.sleep(0.5)
  18. print ""
  19. for listing_page in to_parse:
  20. results, count = webshotslib.get_users(listing_page, False)
  21. for user in results:
  22. if user not in users:
  23. users.append(user)
  24. users_count += 1
  25. print "\rUsers found: %d" % users_count,
  26. if users_count % 1000 < 100 and users_last_save != users_count:
  27. userfile = open("users.txt", "w")
  28. userfile.write("\n".join(users))
  29. userfile.close()
  30. users_last_save = users_count
  31. time.sleep(0.5)
  32. userfile = open("users.txt", "w")
  33. userfile.write("\n".join(users))
  34. userfile.close()