2 Antworten in diesem Thema

[sup3ria]

#Muss noch gevent und requests installiert werden.
#Aufruf: python chkr.py in.txt out.txt threadanzahl
#bsp: python chkr.py urls.txt out.txt 1000

#wird async verarbeitet.

#-*- coding: iso-8859-1 -*-
import gevent # sudo apt-get install python-gevent // pip install gevent
from gevent.queue import Queue
Queue = Queue()
from timeit import default_timer as timer
import requests# pip install requests
import sys

print "URL Status Code Checker - by sup3ria"
print ""

#----------------------------------
if sys.argv[1:]:
    file_in  = sys.argv[1]
else:
	file_in = "in.txt"
	
if sys.argv[2:]:
    file_out  = sys.argv[2]
else:
	file_out = "out.txt"
	
if sys.argv[3:]:
    workers  = int(sys.argv[3])+1
else:
	workers = 11

#----------------------------------

def get(i):
    while not Queue.empty():
            url = Queue.get()
            try:
                r = requests.get(url)
                status = r.status_code
                status_to_write =  str(url).strip()  +" | "+ str(status).strip() 
                print status_to_write   
                write(status_to_write)    
            except:
                pass          
            gevent.sleep(0.1)
            
            

def sender():
    with open(file_in, "rt") as text_file:
        for line in text_file:
            if len(line.strip()) >1:
                Queue.put_nowait(line)

def write(valid):
    with open(file_out, 'a') as file:
        file.write('{}'.format(valid))
        
def asynchronous():
    threads = []
    start = timer()
    for i in range(1,workers):
        threads.append(gevent.spawn(get, i))
    gevent.joinall(threads, timeout=5)
    end = timer()
    return "Time passed: " +str(end - start)[:4]


gevent.spawn(sender).join()
print asynchronous()

[bumg2]

Du machst bei jedem Schreibvorgang das File auf, ist dies gewollt?

[sup3ria]

Du machst bei jedem Schreibvorgang das File auf, ist dies gewollt?

Ja, da die greenlets nicht von einander Dinge erben können.