---
title: "Downloading Files : A Comparative Study"
description: "A simple case study of downloading files in perl, ruby, python and php. Assumption : The input for these scripts are in JS."
url: "https://h3manth.com/scribe/downloading-files-comparative-study/"
canonical_url: "https://h3manth.com/scribe/downloading-files-comparative-study/"
markdown_url: "https://h3manth.com/scribe/blog/downloading-files-comparative-study.md"
publishDate: "2011-06-12T00:00:00.000Z"
author: "Hemanth HM"
category: "python"
tags: ["ruby","python"]
word_count: 621
reading_time_minutes: 4
x_markdown_tokens: 1641
content_signal: "ai-train=yes, search=yes, ai-input=yes"
---

# Downloading Files : A Comparative Study

> A simple case study of downloading files in perl, ruby, python and php. Assumption : The input for these scripts are in JS.

##  The idea : 

A simple case study of downloading files in perl, ruby, python and php.

##  The algorithm : 

_Assumption :_ The input for these scripts are obtained from a simple cgi form.

  * 0\. Hit an URL which in turn responds with a list of URLs ( Like RealMedia Metafiles ) 
  * 1\. Download the contents of each URL that the is returned from the master. 
  * 2\. Zip all the downloaded files and exit. 

##  Downloading files in perl :
    
    
    #!/usr/bin/perl -w
    use strict;
    use warnings; 
    use CGI qw(:standard);
    use LWP::Simple qw(!head);
    use File::Basename;
    use Archive::Zip;
    
    # URL from the input form.
    my $url = param('url');
    
    # Validate the input
    if(defined $url and head($url))
    {
        # Get the ablum name,
        # in this case the second last part.  
        my @pieces = (split /\//,$url);
        my $album = $pieces[-2];
        my @songs = ();
        
        # Fetch the urls
        my $urls = get($url);
       
        # For each url in the url list.
        foreach my $url (split /\n/, $urls) {
        
            # Push the song_names to songs 
            push(@songs,basename($url));
           
            # The major part of downlaading.
            getstore($url,basename($url));
        }   
        
        my $compressor = Archive::Zip->new();
        
        # For each song that was pushed before.
        foreach my $song (@songs)
        {
            # Add them to the zip archive.    
            $compressor->addFile($song);
        }
        
        # Write the zip file.
        $compressor->writeToFileNamed($album.'.zip');
    }
    else
    {
     print "<b> Please check the URL! </b>";
    }
    

##  Downloading files in ruby :
    
    
    #!/usr/bin/ruby
    require 'uri'
    
    urls = cgi.params['url'].to_s
    
    if (!(urls =~ URI::regexp).nil?)
    
       album = urls.split('/') [-2] 
       songs = []
    
       open(urls).read.each_line do |url|
          song = url.split("/")[-1].chomp
          Zip::ZipFile.open(album+'.zip', Zip::ZipFile::CREATE) {
            |zipit|
            zipit.get_output_stream(song) { |f| f.puts open(url.chomp).read }
       } 
       end
    else
       puts "Please check the input!"
    end
    

##  Downloading files in python:
    
    
    import cgi
    import cgitb 
    import urllib
    import zipfile
    
    cgitb.enable()
    
    form = cgi.FieldStorage()
    url = form.getvalue("url", "None")
    
    if url != "None":
        type = url [ -3: ]
        album = url.split('/')[-2]
        try:
            urls = urllib.urlopen(url)  
        except IOError:
            print "Please check the URL"
            exit
    
        songs = []
        
        for url in urls:
        song = url.split("/")[-1]
            urllib.urlretrieve(url,song)
        songs.append(song)
    
        zipit = zipfile.ZipFile(album+'.zip', 'w')    
    
        for song in songs:
            zipit.write(song)
    

##  Downloading files in PHP:
    
    
    <?php
    
    if (isset($params['host']) && $params['host'])      $header[]="Host: " . $params['host'];
    
    /* creates a compressed zip file */
    function create_zip($files = array(),$destination = '',$overwrite = true) {
        
        //avoid over write
        if(file_exists($destination) && !$overwrite) { return false; }
        
        $valid_files = array();
        
        if(is_array($files)) {
            // check each file
            foreach($files as $file) {
                if(file_exists($file)) {
                    $valid_files[] = $file;
                }
            }
        }
        
        if(count($valid_files)) {
            //create the archive
            $zip = new ZipArchive();
            if($zip->open($destination,$overwrite ? ZIPARCHIVE::OVERWRITE : ZIPARCHIVE::CREATE) !== true) {
                return false;
            }
            //add the files
            foreach($valid_files as $file) {
                $zip->addFile($file,$file);
            }
            $zip->close();
            
            return file_exists($destination);
        }
        else
        {
            return false;
        }
    }
    
    function get_data($url)
    {
        $url = chop($url);
        $ch = curl_init();
        $timeout = 5;
        curl_setopt($ch,CURLOPT_URL,$url);
        curl_setopt($ch,CURLOPT_RETURNTRANSFER,1);
        curl_setopt($ch,CURLOPT_CONNECTTIMEOUT,$timeout);
        $data = curl_exec($ch);
        curl_close($ch);
        return $data;
    }
    
    function isValidURL($url)
    {
    return preg_match('|^http(s)?://[a-z0-9-]+(.[a-z0-9-]+)*(:[0-9]+)?(/.*)?$|i', $url);
    }
    
    if(isset($_POST['submit']))
    {
        $url = chop($_POST['url']);
        $songs = array();
       
        $pieces = explode('/',parse_url($url, PHP_URL_PATH)); 
        $album  = $pieces[count($pieces)-2];
        
        if(isValidURL($url) and substr(trim($url), -4) === ".php")
        {
        $url = str_replace(".php","/All.ram", $url);
            $url = str_replace("/home","",$url);
        $album = $pieces[count($pieces) - 1];
        $album = str_replace(".php","",$album);
        }
       
        if(!isValidURL($url) or substr(trim($url), -4) != ".ram"){
            echo "<p>Error! Please check the URL guru!</p>";
        }
     
    
        else{
        $returned_content = get_data($url);
        foreach(explode("\n", get_data($url)) as $dload){
            $song=rtrim(end(explode("/",$dload)));
            if(!empty($song))
            {
                array_push($songs,$song);
                file_put_contents($song,get_data($dload)); 
            }
        }
        $result = create_zip($songs,$album.'.zip');
       
        }
    }
    
    ?>
    

## **The average runtime ( 3 runs ) on a 3MB/s line for a file size of 13MB was :**

Language  | Time in secs  
---|---  
PHP |  11.49   
Perl |  9.0  
Ruby  |  17.22  
Python |  0.24  
  
**All said and done, a simple wget -i took :** real 0m5.231s user 0m0.012s sys 0m0.131s

_P.S : This was a very specific case, this can't be a generic benchmark, there are always better way of tuning the code above!_