| .. | |||||
| __init__.py | |||||
| __init__.pyc | |||||
| __init__.pyo | |||||
| byterange.py | |||||
| byterange.pyc | |||||
| byterange.pyo | |||||
| grabber.py | |||||
| grabber.pyc | |||||
| grabber.pyo | |||||
| mirror.py | |||||
| mirror.pyc | |||||
| mirror.pyo | |||||
| progress.py | |||||
| progress.pyc | |||||
| progress.pyo |
| Server IP : 172.67.171.101 / Your IP : 216.73.216.123 Web Server : Apache System : Linux server1.morocco-tours.com 3.10.0-1127.19.1.el7.x86_64 #1 SMP Tue Aug 25 17:23:54 UTC 2020 x86_64 User : zagoradraa ( 1005) PHP Version : 7.4.33 Disable Function : NONE MySQL : OFF | cURL : ON | WGET : ON | Perl : ON | Python : ON | Sudo : ON | Pkexec : ON Directory : /lib/python2.7/site-packages/urlgrabber/ |
Upload File : |
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
import sys
import time
import math
import thread
import fcntl
import struct
import termios
# Code from http://mail.python.org/pipermail/python-list/2000-May/033365.html
def terminal_width(fd=1):
""" Get the real terminal width """
try:
buf = 'abcdefgh'
buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
ret = struct.unpack('hhhh', buf)[1]
if ret == 0:
return 80
# Add minimum too?
return ret
except: # IOError
return 80
_term_width_val = None
_term_width_last = None
def terminal_width_cached(fd=1, cache_timeout=1.000):
""" Get the real terminal width, but cache it for a bit. """
global _term_width_val
global _term_width_last
now = time.time()
if _term_width_val is None or (now - _term_width_last) > cache_timeout:
_term_width_val = terminal_width(fd)
_term_width_last = now
return _term_width_val
class TerminalLine:
""" Help create dynamic progress bars, uses terminal_width_cached(). """
def __init__(self, min_rest=0, beg_len=None, fd=1, cache_timeout=1.000):
if beg_len is None:
beg_len = min_rest
self._min_len = min_rest
self._llen = terminal_width_cached(fd, cache_timeout)
if self._llen < beg_len:
self._llen = beg_len
self._fin = False
def __len__(self):
""" Usable length for elements. """
return self._llen - self._min_len
def rest_split(self, fixed, elements=2):
""" After a fixed length, split the rest of the line length among
a number of different elements (default=2). """
if self._llen < fixed:
return 0
return (self._llen - fixed) / elements
def add(self, element, full_len=None):
""" If there is room left in the line, above min_len, add element.
Note that as soon as one add fails all the rest will fail too. """
if full_len is None:
full_len = len(element)
if len(self) < full_len:
self._fin = True
if self._fin:
return ''
self._llen -= len(element)
return element
def rest(self):
""" Current rest of line, same as .rest_split(fixed=0, elements=1). """
return self._llen
class BaseMeter:
def __init__(self):
self.update_period = 0.3 # seconds
self.filename = None
self.url = None
self.basename = None
self.text = None
self.size = None
self.start_time = None
self.last_amount_read = 0
self.last_update_time = None
self.re = RateEstimator()
def start(self, filename=None, url=None, basename=None,
size=None, now=None, text=None):
self.filename = filename
self.url = url
self.basename = basename
self.text = text
#size = None ######### TESTING
self.size = size
if not size is None: self.fsize = format_number(size) + 'B'
if now is None: now = time.time()
self.start_time = now
self.re.start(size, now)
self.last_amount_read = 0
self.last_update_time = now
self._do_start(now)
def _do_start(self, now=None):
pass
def update(self, amount_read, now=None):
# for a real gui, you probably want to override and put a call
# to your mainloop iteration function here
if now is None: now = time.time()
if (not self.last_update_time or
(now >= self.last_update_time + self.update_period)):
self.re.update(amount_read, now)
self.last_amount_read = amount_read
self.last_update_time = now
self._do_update(amount_read, now)
def _do_update(self, amount_read, now=None):
pass
def end(self, amount_read, now=None):
if now is None: now = time.time()
self.re.update(amount_read, now)
self.last_amount_read = amount_read
self.last_update_time = now
self._do_end(amount_read, now)
def _do_end(self, amount_read, now=None):
pass
# This is kind of a hack, but progress is gotten from grabber which doesn't
# know about the total size to download. So we do this so we can get the data
# out of band here. This will be "fixed" one way or anther soon.
_text_meter_total_size = 0
_text_meter_sofar_size = 0
def text_meter_total_size(size, downloaded=0):
global _text_meter_total_size
global _text_meter_sofar_size
_text_meter_total_size = size
_text_meter_sofar_size = downloaded
#
# update: No size (minimal: 17 chars)
# -----------------------------------
# <text> <rate> | <current size> <elapsed time>
# 8-48 1 8 3 6 1 9 5
#
# Order: 1. <text>+<current size> (17)
# 2. +<elapsed time> (10, total: 27)
# 3. + ( 5, total: 32)
# 4. +<rate> ( 9, total: 41)
#
# update: Size, Single file
# -------------------------
# <text> <pc> <bar> <rate> | <current size> <eta time> ETA
# 8-25 1 3-4 1 6-16 1 8 3 6 1 9 1 3 1
#
# Order: 1. <text>+<current size> (17)
# 2. +<eta time> (10, total: 27)
# 3. +ETA ( 5, total: 32)
# 4. +<pc> ( 4, total: 36)
# 5. +<rate> ( 9, total: 45)
# 6. +<bar> ( 7, total: 52)
#
# update: Size, All files
# -----------------------
# <text> <total pc> <pc> <bar> <rate> | <current size> <eta time> ETA
# 8-22 1 5-7 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1
#
# Order: 1. <text>+<current size> (17)
# 2. +<eta time> (10, total: 27)
# 3. +ETA ( 5, total: 32)
# 4. +<total pc> ( 5, total: 37)
# 4. +<pc> ( 4, total: 41)
# 5. +<rate> ( 9, total: 50)
# 6. +<bar> ( 7, total: 57)
#
# end
# ---
# <text> | <current size> <elapsed time>
# 8-56 3 6 1 9 5
#
# Order: 1. <text> ( 8)
# 2. +<current size> ( 9, total: 17)
# 3. +<elapsed time> (10, total: 27)
# 4. + ( 5, total: 32)
#
def _term_add_bar(tl, bar_max_length, pc):
blen = bar_max_length
bar = '='*int(blen * pc)
if (blen * pc) - int(blen * pc) >= 0.5:
bar += '-'
return tl.add(' [%-*.*s]' % (blen, blen, bar))
def _term_add_end(tl, osize, size):
if osize: # osize should be None or >0, but that's been broken.
if size > osize: # Is ??? better? Really need something to say < vs >.
return tl.add(' !!! '), True
elif size != osize:
return tl.add(' ... '), True
return tl.add(' ' * 5), False
class TextMeter(BaseMeter):
def __init__(self, fo=sys.stderr):
BaseMeter.__init__(self)
self.fo = fo
def _do_update(self, amount_read, now=None):
etime = self.re.elapsed_time()
fread = format_number(amount_read)
#self.size = None
if self.text is not None:
text = self.text
else:
text = self.basename
ave_dl = format_number(self.re.average_rate())
sofar_size = None
if _text_meter_total_size:
sofar_size = _text_meter_sofar_size + amount_read
sofar_pc = (sofar_size * 100) / _text_meter_total_size
# Include text + ui_rate in minimal
tl = TerminalLine(8, 8+1+8)
if tl._llen > 80:
use_hours = True # For big screens, make it more readable.
else:
use_hours = False
ui_size = tl.add(' | %5sB' % fread)
if self.size is None:
ui_time = tl.add(' %9s' % format_time(etime, use_hours))
ui_end = tl.add(' ' * 5)
ui_rate = tl.add(' %5sB/s' % ave_dl)
out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
ui_rate, ui_size, ui_time, ui_end)
else:
rtime = self.re.remaining_time()
frtime = format_time(rtime, use_hours)
frac = self.re.fraction_read()
ui_time = tl.add(' %9s' % frtime)
ui_end = tl.add(' ETA ')
if sofar_size is None:
ui_sofar_pc = ''
else:
ui_sofar_pc = tl.add(' (%i%%)' % sofar_pc,
full_len=len(" (100%)"))
ui_pc = tl.add(' %2i%%' % (frac*100))
ui_rate = tl.add(' %5sB/s' % ave_dl)
# Make text grow a bit before we start growing the bar too
blen = 4 + tl.rest_split(8 + 8 + 4)
ui_bar = _term_add_bar(tl, blen, frac)
out = '\r%-*.*s%s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
ui_sofar_pc, ui_pc, ui_bar,
ui_rate,ui_size,ui_time, ui_end)
self.fo.write(out)
self.fo.flush()
def _do_end(self, amount_read, now=None):
global _text_meter_total_size
global _text_meter_sofar_size
total_size = format_number(amount_read)
if self.text is not None:
text = self.text
else:
text = self.basename
tl = TerminalLine(8)
if tl._llen > 80:
use_hours = True # For big screens, make it more readable.
else:
use_hours = False
ui_size = tl.add(' | %5sB' % total_size)
ui_time = tl.add(' %9s' % format_time(self.re.elapsed_time(),use_hours))
ui_end, not_done = _term_add_end(tl, self.size, amount_read)
out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
ui_size, ui_time, ui_end)
self.fo.write(out)
self.fo.flush()
# Don't add size to the sofar size until we have all of it.
# If we don't have a size, then just pretend/hope we got all of it.
if not_done:
return
if _text_meter_total_size:
_text_meter_sofar_size += amount_read
if _text_meter_total_size <= _text_meter_sofar_size:
_text_meter_total_size = 0
_text_meter_sofar_size = 0
text_progress_meter = TextMeter
class MultiFileHelper(BaseMeter):
def __init__(self, master):
BaseMeter.__init__(self)
self.master = master
def _do_start(self, now):
self.master.start_meter(self, now)
def _do_update(self, amount_read, now):
# elapsed time since last update
self.master.update_meter(self, now)
def _do_end(self, amount_read, now):
self.ftotal_time = format_time(now - self.start_time)
self.ftotal_size = format_number(self.last_amount_read)
self.master.end_meter(self, now)
def failure(self, message, now=None):
self.master.failure_meter(self, message, now)
def message(self, message):
self.master.message_meter(self, message)
class _FakeLock:
def acquire(self):
pass
def release(self):
pass
class MultiFileMeter:
helperclass = MultiFileHelper
def __init__(self, threaded=True):
self.meters = []
self.in_progress_meters = []
if threaded:
self._lock = thread.allocate_lock()
else:
self._lock = _FakeLock()
self.update_period = 0.3 # seconds
self.numfiles = None
self.finished_files = 0
self.failed_files = 0
self.open_files = 0
self.total_size = None
self.failed_size = 0
self.start_time = None
self.finished_file_size = 0
self.last_update_time = None
self.re = RateEstimator()
def start(self, numfiles=None, total_size=None, now=None):
if now is None: now = time.time()
self.numfiles = numfiles
self.finished_files = 0
self.failed_files = 0
self.open_files = 0
self.total_size = total_size
self.failed_size = 0
self.start_time = now
self.finished_file_size = 0
self.last_update_time = now
self.re.start(total_size, now)
self._do_start(now)
def _do_start(self, now):
pass
def end(self, now=None):
if now is None: now = time.time()
self.re.update(self._amount_read(), now)
self._do_end(now)
def _do_end(self, now):
pass
def lock(self): self._lock.acquire()
def unlock(self): self._lock.release()
###########################################################
# child meter creation and destruction
def newMeter(self):
newmeter = self.helperclass(self)
self.meters.append(newmeter)
return newmeter
def removeMeter(self, meter):
self.meters.remove(meter)
###########################################################
# child functions - these should only be called by helpers
def start_meter(self, meter, now):
if not meter in self.meters:
raise ValueError('attempt to use orphaned meter')
self._lock.acquire()
try:
if not meter in self.in_progress_meters:
self.in_progress_meters.append(meter)
self.open_files += 1
finally:
self._lock.release()
self._do_start_meter(meter, now)
def _do_start_meter(self, meter, now):
pass
def update_meter(self, meter, now):
if not meter in self.meters:
raise ValueError('attempt to use orphaned meter')
if (not self.last_update_time or
(now >= self.last_update_time + self.update_period)):
self.re.update(self._amount_read(), now)
self.last_update_time = now
self._do_update_meter(meter, now)
def _do_update_meter(self, meter, now):
pass
def end_meter(self, meter, now):
if not meter in self.meters:
raise ValueError('attempt to use orphaned meter')
self._lock.acquire()
try:
try: self.in_progress_meters.remove(meter)
except ValueError: pass
self.open_files -= 1
self.finished_files += 1
self.finished_file_size += meter.last_amount_read
finally:
self._lock.release()
self._do_end_meter(meter, now)
def _do_end_meter(self, meter, now):
pass
def failure_meter(self, meter, message, now):
if not meter in self.meters:
raise ValueError('attempt to use orphaned meter')
self._lock.acquire()
try:
try: self.in_progress_meters.remove(meter)
except ValueError: pass
self.open_files -= 1
self.failed_files += 1
if meter.size and self.failed_size is not None:
self.failed_size += meter.size
else:
self.failed_size = None
finally:
self._lock.release()
self._do_failure_meter(meter, message, now)
def _do_failure_meter(self, meter, message, now):
pass
def message_meter(self, meter, message):
pass
########################################################
# internal functions
def _amount_read(self):
tot = self.finished_file_size
for m in self.in_progress_meters:
tot += m.last_amount_read
return tot
class TextMultiFileMeter(MultiFileMeter):
def __init__(self, fo=sys.stderr, threaded=True):
self.fo = fo
MultiFileMeter.__init__(self, threaded)
self.index_time = self.index = 0
# files: ###/### ###% data: ######/###### ###% time: ##:##:##/##:##:##
# New output, like TextMeter output...
# update: No size (minimal: 17 chars)
# -----------------------------------
# (<#file>/<#tot files>): <text> <rate> | <current size> <elapsed>
# 8-48 1 8 3 6 1 7-9 5
#
# update: Size, All files
# -----------------------
# (<#file>/<#tot files>): <text> <pc> <bar> <rate> | <size> <eta time> ETA
# 8-22 1 3-4 1 6-12 1 8 3 6 1 7-9 1 3 1
# end
# ---
# <text> | <file size> <file elapsed time>
# 8-56 3 6 1 9 5
def _do_update_meter(self, meter, now):
self._lock.acquire()
try:
df = self.finished_files
tf = self.numfiles or 1
# Don't use "percent of files complete" ...
# pf = 100 * float(df)/tf + 0.49
dd = self.re.last_amount_read
td = self.re.total
pd = 100 * (self.re.fraction_read() or 0) + 0.49
dt = self.re.elapsed_time()
rt = self.re.remaining_time()
frac = self.re.fraction_read() or 0
pf = 100 * frac
ave_dl = format_number(self.re.average_rate())
# cycle through active meters
if now > self.index_time:
self.index_time = now + 1.0
self.index += 1
if self.index >= len(self.meters):
self.index = 0
meter = self.meters[self.index]
text = meter.text or meter.basename
if tf > 1:
text = '(%u/%u): %s' % (df+1+self.index, tf, text)
# Include text + ui_rate in minimal
tl = TerminalLine(8, 8+1+8)
if tl._llen > 80:
use_hours = True # For big screens, make it more readable.
time_len = 9
else:
use_hours = False
time_len = 7
ui_size = tl.add(' | %5sB' % format_number(dd))
if not self.re.total:
ui_time = tl.add(' %*s' % (time_len,format_time(dt, use_hours)))
ui_end = tl.add(' ' * 5)
ui_rate = tl.add(' %5sB/s' % ave_dl)
out = '\r%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
ui_rate, ui_size, ui_time, ui_end)
else:
ui_time = tl.add(' %*s' % (time_len,format_time(rt, use_hours)))
ui_end = tl.add(' ETA ')
ui_sofar_pc = tl.add(' %i%%' % pf,
full_len=len(" (100%)"))
ui_rate = tl.add(' %5sB/s' % ave_dl)
# Make text grow a bit before we start growing the bar too
blen = 4 + tl.rest_split(8 + 8 + 4)
ui_bar = _term_add_bar(tl, blen, frac)
out = '\r%-*.*s%s%s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
ui_sofar_pc, ui_bar,
ui_rate, ui_size, ui_time,
ui_end)
self.fo.write(out)
self.fo.flush()
finally:
self._lock.release()
def _do_end_meter(self, meter, now):
self._lock.acquire()
try:
format = "%-30.30s %6.6s %8.8s %9.9s"
fn = meter.text or meter.basename
size = meter.last_amount_read
fsize = format_number(size) + 'B'
et = meter.re.elapsed_time()
frate = format_number(et and size / et) + 'B/s'
df = self.finished_files
tf = self.numfiles or 1
total_size = format_number(size)
text = meter.text or meter.basename
if tf > 1:
text = '(%u/%u): %s' % (df, tf, text)
tl = TerminalLine(8)
if tl._llen > 80:
use_hours = True # For big screens, make it more readable.
time_len = 9
else:
use_hours = False
time_len = 7
ui_size = tl.add(' | %5sB' % total_size)
ui_time = tl.add(' %*s' % (time_len, format_time(et, use_hours)))
ui_end, not_done = _term_add_end(tl, meter.size, size)
out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
ui_size, ui_time, ui_end)
self.fo.write(out)
finally:
self._lock.release()
def _do_failure_meter(self, meter, message, now):
self._lock.acquire()
try:
format = "%-30.30s %6.6s %s"
fn = meter.text or meter.basename
if type(message) in (type(''), type(u'')):
message = message.splitlines()
if not message: message = ['']
out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
self.fo.write('\r' + out + '\n')
for m in message[1:]: self.fo.write(' ' + m + '\n')
self._lock.release()
finally:
self._do_update_meter(meter, now)
def message_meter(self, meter, message):
self._lock.acquire()
try:
pass
finally:
self._lock.release()
######################################################################
# support classes and functions
class RateEstimator:
def __init__(self, timescale=5.0):
self.timescale = timescale
def start(self, total=None, now=None):
if now is None: now = time.time()
self.total = total
self.start_time = now
self.last_update_time = now
self.last_amount_read = 0
self.ave_rate = None
def update(self, amount_read, now=None):
if now is None: now = time.time()
# libcurl calls the progress callback when fetching headers
# too, thus amount_read = 0 .. hdr_size .. 0 .. content_size.
# Ocassionally we miss the 2nd zero and report avg speed < 0.
# Handle read_diff < 0 here. BZ 1001767.
if amount_read == 0 or amount_read < self.last_amount_read:
# if we just started this file, all bets are off
self.last_update_time = now
self.last_amount_read = amount_read
self.ave_rate = None
return
#print 'times', now, self.last_update_time
time_diff = now - self.last_update_time
read_diff = amount_read - self.last_amount_read
# First update, on reget is the file size
if self.last_amount_read:
self.last_update_time = now
self.ave_rate = self._temporal_rolling_ave(\
time_diff, read_diff, self.ave_rate, self.timescale)
self.last_amount_read = amount_read
#print 'results', time_diff, read_diff, self.ave_rate
#####################################################################
# result methods
def average_rate(self):
"get the average transfer rate (in bytes/second)"
return self.ave_rate
def elapsed_time(self):
"the time between the start of the transfer and the most recent update"
return self.last_update_time - self.start_time
def remaining_time(self):
"estimated time remaining"
if not self.ave_rate or not self.total: return None
return (self.total - self.last_amount_read) / self.ave_rate
def fraction_read(self):
"""the fraction of the data that has been read
(can be None for unknown transfer size)"""
if self.total is None: return None
elif self.total == 0: return 1.0
else: return float(self.last_amount_read)/self.total
#########################################################################
# support methods
def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
"""a temporal rolling average performs smooth averaging even when
updates come at irregular intervals. This is performed by scaling
the "epsilon" according to the time since the last update.
Specifically, epsilon = time_diff / timescale
As a general rule, the average will take on a completely new value
after 'timescale' seconds."""
epsilon = time_diff / timescale
if epsilon > 1: epsilon = 1.0
return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
"""perform a "rolling average" iteration
a rolling average "folds" new data into an existing average with
some weight, epsilon. epsilon must be between 0.0 and 1.0 (inclusive)
a value of 0.0 means only the old value (initial value) counts,
and a value of 1.0 means only the newest value is considered."""
try:
recent_rate = read_diff / time_diff
except ZeroDivisionError:
recent_rate = None
if last_ave is None: return recent_rate
elif recent_rate is None: return last_ave
# at this point, both last_ave and recent_rate are numbers
return epsilon * recent_rate + (1 - epsilon) * last_ave
def _round_remaining_time(self, rt, start_time=15.0):
"""round the remaining time, depending on its size
If rt is between n*start_time and (n+1)*start_time round downward
to the nearest multiple of n (for any counting number n).
If rt < start_time, round down to the nearest 1.
For example (for start_time = 15.0):
2.7 -> 2.0
25.2 -> 25.0
26.4 -> 26.0
35.3 -> 34.0
63.6 -> 60.0
"""
if rt < 0: return 0.0
shift = int(math.log(rt/start_time)/math.log(2))
rt = int(rt)
if shift <= 0: return rt
return float(int(rt) >> shift << shift)
def format_time(seconds, use_hours=0):
if seconds is None or seconds < 0:
if use_hours: return '--:--:--'
else: return '--:--'
elif seconds == float('inf'):
return 'Infinite'
else:
seconds = int(seconds)
minutes = seconds / 60
seconds = seconds % 60
if use_hours:
hours = minutes / 60
minutes = minutes % 60
return '%02i:%02i:%02i' % (hours, minutes, seconds)
else:
return '%02i:%02i' % (minutes, seconds)
def format_number(number, SI=0, space=' '):
"""Turn numbers into human-readable metric-like numbers"""
symbols = ['', # (none)
'k', # kilo
'M', # mega
'G', # giga
'T', # tera
'P', # peta
'E', # exa
'Z', # zetta
'Y'] # yotta
if SI: step = 1000.0
else: step = 1024.0
thresh = 999
depth = 0
max_depth = len(symbols) - 1
# we want numbers between 0 and thresh, but don't exceed the length
# of our list. In that event, the formatting will be screwed up,
# but it'll still show the right number.
while number > thresh and depth < max_depth:
depth = depth + 1
number = number / step
if type(number) == type(1) or type(number) == type(1L):
# it's an int or a long, which means it didn't get divided,
# which means it's already short enough
format = '%i%s%s'
elif number < 9.95:
# must use 9.95 for proper sizing. For example, 9.99 will be
# rounded to 10.0 with the .1f format string (which is too long)
format = '%.1f%s%s'
else:
format = '%.0f%s%s'
return(format % (float(number or 0), space, symbols[depth]))
def _tst(fn, cur, tot, beg, size, *args):
tm = TextMeter()
text = "(%d/%d): %s" % (cur, tot, fn)
tm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size, text=text)
num = beg
off = 0
for (inc, delay) in args:
off += 1
while num < ((size * off) / len(args)):
num += inc
tm.update(num)
time.sleep(delay)
tm.end(size)
def _mtst(datas, *args):
print '-' * 79
tm = TextMultiFileMeter(threaded=False)
dl_sizes = {}
num = 0
total_size = 0
dl_total_size = 0
for data in datas:
dl_size = None
if len(data) == 2:
fn, size = data
dl_size = size
if len(data) == 3:
fn, size, dl_size = data
nm = tm.newMeter()
nm.start(fn, "http://www.example.com/path/to/fn/" + fn, fn, size,
text=fn)
num += 1
assert dl_size is not None
dl_total_size += dl_size
dl_sizes[nm] = dl_size
if size is None or total_size is None:
total_size = None
else:
total_size += size
tm.start(num, total_size)
num = 0
off = 0
for (inc, delay) in args:
off += 1
while num < ((dl_total_size * off) / len(args)):
num += inc
for nm in tm.meters[:]:
if dl_sizes[nm] <= num:
nm.end(dl_sizes[nm])
tm.removeMeter(nm)
else:
nm.update(num)
time.sleep(delay)
assert not tm.meters
if __name__ == "__main__":
# (1/2): subversion-1.4.4-7.x86_64.rpm 2.4 MB / 85 kB/s 00:28
# (2/2): mercurial-0.9.5-6.fc8.x86_64.rpm 924 kB / 106 kB/s 00:08
if len(sys.argv) >= 2 and sys.argv[1] == 'multi':
_mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
("s-1.0.1-1.fc8.i386.rpm", 5000),
("m-1.0.1-2.fc8.i386.rpm", 10000)),
(100, 0.33), (500, 0.25), (1000, 0.1))
_mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
("s-1.0.1-1.fc8.i386.rpm", 5000),
("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
(100, 0.33), (500, 0.25), (1000, 0.1))
_mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
("s-1.0.1-1.fc8.i386.rpm", 2500000),
("m-1.0.1-2.fc8.i386.rpm", 10000)),
(10, 0.2), (50, 0.1), (1000, 0.1))
_mtst((("sm-1.0.0-1.fc8.i386.rpm", 1000),
("s-1.0.1-1.fc8.i386.rpm", None, 2500000),
("m-1.0.1-2.fc8.i386.rpm", None, 10000)),
(10, 0.2), (50, 0.1), (1000, 0.1))
# (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
# (10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
sys.exit(0)
if len(sys.argv) >= 2 and sys.argv[1] == 'total':
text_meter_total_size(1000 + 10000 + 10000 + 1000000 + 1000000 +
1000000 + 10000 + 10000 + 10000 + 1000000)
_tst("sm-1.0.0-1.fc8.i386.rpm", 1, 10, 0, 1000,
(10, 0.2), (10, 0.1), (100, 0.25))
_tst("s-1.0.1-1.fc8.i386.rpm", 2, 10, 0, 10000,
(10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
_tst("m-1.0.1-2.fc8.i386.rpm", 3, 10, 5000, 10000,
(10, 0.2), (100, 0.1), (100, 0.1), (100, 0.25))
_tst("large-file-name-Foo-11.8.7-4.5.6.1.fc8.x86_64.rpm", 4, 10, 0, 1000000,
(1000, 0.2), (1000, 0.1), (10000, 0.1))
_tst("large-file-name-Foo2-11.8.7-4.5.6.2.fc8.x86_64.rpm", 5, 10,
500001, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
_tst("large-file-name-Foo3-11.8.7-4.5.6.3.fc8.x86_64.rpm", 6, 10,
750002, 1000000, (1000, 0.2), (1000, 0.1), (10000, 0.1))
_tst("large-file-name-Foo4-10.8.7-4.5.6.1.fc8.x86_64.rpm", 7, 10, 0, 10000,
(100, 0.1))
_tst("large-file-name-Foo5-10.8.7-4.5.6.2.fc8.x86_64.rpm", 8, 10,
5001, 10000, (100, 0.1))
_tst("large-file-name-Foo6-10.8.7-4.5.6.3.fc8.x86_64.rpm", 9, 10,
7502, 10000, (1, 0.1))
_tst("large-file-name-Foox-9.8.7-4.5.6.1.fc8.x86_64.rpm", 10, 10,
0, 1000000, (10, 0.5),
(100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
(100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
(100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
(100000, 0.1), (10000, 0.1), (10000, 0.1), (10000, 0.1),
(100000, 0.1), (1, 0.1))
| .. | |||||
| __init__.py | |||||
| __init__.pyc | |||||
| __init__.pyo | |||||
| byterange.py | |||||
| byterange.pyc | |||||
| byterange.pyo | |||||
| grabber.py | |||||
| grabber.pyc | |||||
| grabber.pyo | |||||
| mirror.py | |||||
| mirror.pyc | |||||
| mirror.pyo | |||||
| progress.py | |||||
| progress.pyc | |||||
| progress.pyo |