How to script FTP upload and download
It's a reasonable idea to want to script an FTP session the way the original poster imagined, and that is the kind of thing Expect would help with. Batch files on Windows cannot do this.
But rather than doing cURL or Expect, you may find it easier to script the FTP interaction with PowerShell. It's a different model, in that you are not directly scripting the text to send to the FTP server. Instead you will use PowerShell to manipulate objects that generate the FTP dialogue for you.
Upload:
$File = "D:\Dev\somefilename.zip"
$ftp = "ftp://username:[email protected]/pub/incoming/somefilename.zip"
"ftp url: $ftp"
$webclient = New-Object System.Net.WebClient
$uri = New-Object System.Uri($ftp)
"Uploading $File..."
$webclient.UploadFile($uri, $File)
Download:
$File = "c:\store\somefilename.zip"
$ftp = "ftp://username:[email protected]/pub/outbound/somefilename.zip"
"ftp url: $ftp"
$webclient = New-Object System.Net.WebClient
$uri = New-Object System.Uri($ftp)
"Downloading $File..."
$webclient.DownloadFile($uri, $File)
You need PowerShell to do this. If you are not aware, PowerShell is a shell like cmd.exe which runs your .bat files. But PowerShell runs .ps1 files, and is quite a bit more powerful. PowerShell is a free add-on to Windows and will be built-in to future versions of Windows. Get it here.
Source: http://poshcode.org/1134
Create a command file with your commands.
I.e., file commands.txt:
open www.domainhere.com
user useridhere
passwordhere
put test.txt
bye
Then run the FTP client from the command line:
ftp -s:commands.txt
Note: This will work for the Windows FTP client.
Batch files don't work that way. They don't just "type" everything - they run system commands, in this case ftp, wait for them to return, and run the next command... so in this case, the interpreter is simply waiting for ftp to exit.
If you must use the ftp command, then prepare a script file (for example, commands.txt and run ftp -s:commands.txt.
But using cURL, or a PHP/Perl/Python/whatever script may be a better idea.
I've done this with PowerShell:
function DownloadFromFtp($destination, $ftp_uri, $user, $pass){
$dirs = GetDirecoryTree $ftp_uri $user $pass
foreach($dir in $dirs){
$path = [io.path]::Combine($destination,$dir)
if ((Test-Path $path) -eq $false) {
"Creating $path ..."
New-Item -Path $path -ItemType Directory | Out-Null
}else{
"Exists $path ..."
}
}
$files = GetFilesTree $ftp_uri $user $pass
foreach($file in $files){
$source = [io.path]::Combine($ftp_uri,$file)
$dest = [io.path]::Combine($destination,$file)
"Downloading $source ..."
Get-FTPFile $source $dest $user $pass
}
}
function UploadToFtp($artifacts, $ftp_uri, $user, $pass){
$webclient = New-Object System.Net.WebClient
$webclient.Credentials = New-Object System.Net.NetworkCredential($user,$pass)
foreach($item in Get-ChildItem -recurse $artifacts){
$relpath = [system.io.path]::GetFullPath($item.FullName).SubString([system.io.path]::GetFullPath($artifacts).Length + 1)
if ($item.Attributes -eq "Directory"){
try{
Write-Host Creating $item.Name
$makeDirectory = [System.Net.WebRequest]::Create($ftp_uri+$relpath);
$makeDirectory.Credentials = New-Object System.Net.NetworkCredential($user,$pass)
$makeDirectory.Method = [System.Net.WebRequestMethods+FTP]::MakeDirectory;
$makeDirectory.GetResponse();
}catch [Net.WebException] {
Write-Host $item.Name probably exists ...
}
continue;
}
"Uploading $item..."
$uri = New-Object System.Uri($ftp_uri+$relpath)
$webclient.UploadFile($uri, $item.FullName)
}
}
function Get-FTPFile ($Source,$Target,$UserName,$Password)
{
$ftprequest = [System.Net.FtpWebRequest]::create($Source)
$ftprequest.Credentials = New-Object System.Net.NetworkCredential($username,$password)
$ftprequest.Method = [System.Net.WebRequestMethods+Ftp]::DownloadFile
$ftprequest.UseBinary = $true
$ftprequest.KeepAlive = $false
$ftpresponse = $ftprequest.GetResponse()
$responsestream = $ftpresponse.GetResponseStream()
$targetfile = New-Object IO.FileStream ($Target,[IO.FileMode]::Create)
[byte[]]$readbuffer = New-Object byte[] 1024
do{
$readlength = $responsestream.Read($readbuffer,0,1024)
$targetfile.Write($readbuffer,0,$readlength)
}
while ($readlength -ne 0)
$targetfile.close()
}
#task ListFiles {
#
# $files = GetFilesTree 'ftp://127.0.0.1/' "web" "web"
# $files | ForEach-Object {Write-Host $_ -foregroundcolor cyan}
#}
function GetDirecoryTree($ftp, $user, $pass){
$creds = New-Object System.Net.NetworkCredential($user,$pass)
$files = New-Object "system.collections.generic.list[string]"
$folders = New-Object "system.collections.generic.queue[string]"
$folders.Enqueue($ftp)
while($folders.Count -gt 0){
$fld = $folders.Dequeue()
$newFiles = GetAllFiles $creds $fld
$dirs = GetDirectories $creds $fld
foreach ($line in $dirs){
$dir = @($newFiles | Where { $line.EndsWith($_) })[0]
[void]$newFiles.Remove($dir)
$folders.Enqueue($fld + $dir + "/")
[void]$files.Add($fld.Replace($ftp, "") + $dir + "/")
}
}
return ,$files
}
function GetFilesTree($ftp, $user, $pass){
$creds = New-Object System.Net.NetworkCredential($user,$pass)
$files = New-Object "system.collections.generic.list[string]"
$folders = New-Object "system.collections.generic.queue[string]"
$folders.Enqueue($ftp)
while($folders.Count -gt 0){
$fld = $folders.Dequeue()
$newFiles = GetAllFiles $creds $fld
$dirs = GetDirectories $creds $fld
foreach ($line in $dirs){
$dir = @($newFiles | Where { $line.EndsWith($_) })[0]
[void]$newFiles.Remove($dir)
$folders.Enqueue($fld + $dir + "/")
}
$newFiles | ForEach-Object {
$files.Add($fld.Replace($ftp, "") + $_)
}
}
return ,$files
}
function GetDirectories($creds, $fld){
$dirs = New-Object "system.collections.generic.list[string]"
$operation = [System.Net.WebRequestMethods+Ftp]::ListDirectoryDetails
$reader = GetStream $creds $fld $operation
while (($line = $reader.ReadLine()) -ne $null) {
if ($line.Trim().ToLower().StartsWith("d") -or $line.Contains(" <DIR> ")) {
[void]$dirs.Add($line)
}
}
$reader.Dispose();
return ,$dirs
}
function GetAllFiles($creds, $fld){
$newFiles = New-Object "system.collections.generic.list[string]"
$operation = [System.Net.WebRequestMethods+Ftp]::ListDirectory
$reader = GetStream $creds $fld $operation
while (($line = $reader.ReadLine()) -ne $null) {
[void]$newFiles.Add($line.Trim())
}
$reader.Dispose();
return ,$newFiles
}
function GetStream($creds, $url, $meth){
$ftp = [System.Net.WebRequest]::Create($url)
$ftp.Credentials = $creds
$ftp.Method = $meth
$response = $ftp.GetResponse()
return New-Object IO.StreamReader $response.GetResponseStream()
}
Export-ModuleMember UploadToFtp, DownLoadFromFtp
You can script the ftp command with the -s:filename option. The syntax is just a list of commands to pass to the ftp shell, each terminated by a newline. This page has a nice reference to the commands that can be performed with ftp.
Upload/Download Entire Directory Structure
Using the normal ftp doesn't work very well when you need to have an entire directory tree copied to or from a FTP site. So you could use something like these to handle those situations.
These scripts work with the Windows ftp command and allows for uploading and downloading of entire directories from a single command. This makes it pretty self-reliant when using on different systems.
Basically, they map out the directory structure to be up/downloaded, dump corresponding ftp commands to a file, and then execute those commands when the mapping has finished.
ftpupload.bat
@echo off
SET FTPADDRESS=%1
SET FTPUSERNAME=%2
SET FTPPASSWORD=%3
SET LOCALDIR=%~f4
SET REMOTEDIR=%5
if "%FTPADDRESS%" == "" goto FTP_UPLOAD_USAGE
if "%FTPUSERNAME%" == "" goto FTP_UPLOAD_USAGE
if "%FTPPASSWORD%" == "" goto FTP_UPLOAD_USAGE
if "%LOCALDIR%" == "" goto FTP_UPLOAD_USAGE
if "%REMOTEDIR%" == "" goto FTP_UPLOAD_USAGE
:TEMP_NAME
set TMPFILE=%TMP%\%RANDOM%_ftpupload.tmp
if exist "%TMPFILE%" goto TEMP_NAME
SET INITIALDIR=%CD%
echo user %FTPUSERNAME% %FTPPASSWORD% > %TMPFILE%
echo bin >> %TMPFILE%
echo lcd %LOCALDIR% >> %TMPFILE%
cd %LOCALDIR%
setlocal EnableDelayedExpansion
echo mkdir !REMOTEDIR! >> !TMPFILE!
echo cd %REMOTEDIR% >> !TMPFILE!
echo mput * >> !TMPFILE!
for /d /r %%d in (*) do (
set CURRENT_DIRECTORY=%%d
set RELATIVE_DIRECTORY=!CURRENT_DIRECTORY:%LOCALDIR%=!
echo mkdir "!REMOTEDIR!/!RELATIVE_DIRECTORY:~1!" >> !TMPFILE!
echo cd "!REMOTEDIR!/!RELATIVE_DIRECTORY:~1!" >> !TMPFILE!
echo mput "!RELATIVE_DIRECTORY:~1!\*" >> !TMPFILE!
)
echo quit >> !TMPFILE!
endlocal EnableDelayedExpansion
ftp -n -i "-s:%TMPFILE%" %FTPADDRESS%
del %TMPFILE%
cd %INITIALDIR%
goto FTP_UPLOAD_EXIT
:FTP_UPLOAD_USAGE
echo Usage: ftpupload [address] [username] [password] [local directory] [remote directory]
echo.
:FTP_UPLOAD_EXIT
set INITIALDIR=
set FTPADDRESS=
set FTPUSERNAME=
set FTPPASSWORD=
set LOCALDIR=
set REMOTEDIR=
set TMPFILE=
set CURRENT_DIRECTORY=
set RELATIVE_DIRECTORY=
@echo on
ftpget.bat
@echo off
SET FTPADDRESS=%1
SET FTPUSERNAME=%2
SET FTPPASSWORD=%3
SET LOCALDIR=%~f4
SET REMOTEDIR=%5
SET REMOTEFILE=%6
if "%FTPADDRESS%" == "" goto FTP_UPLOAD_USAGE
if "%FTPUSERNAME%" == "" goto FTP_UPLOAD_USAGE
if "%FTPPASSWORD%" == "" goto FTP_UPLOAD_USAGE
if "%LOCALDIR%" == "" goto FTP_UPLOAD_USAGE
if not defined REMOTEDIR goto FTP_UPLOAD_USAGE
if not defined REMOTEFILE goto FTP_UPLOAD_USAGE
:TEMP_NAME
set TMPFILE=%TMP%\%RANDOM%_ftpupload.tmp
if exist "%TMPFILE%" goto TEMP_NAME
echo user %FTPUSERNAME% %FTPPASSWORD% > %TMPFILE%
echo bin >> %TMPFILE%
echo lcd %LOCALDIR% >> %TMPFILE%
echo cd "%REMOTEDIR%" >> %TMPFILE%
echo mget "%REMOTEFILE%" >> %TMPFILE%
echo quit >> %TMPFILE%
ftp -n -i "-s:%TMPFILE%" %FTPADDRESS%
del %TMPFILE%
goto FTP_UPLOAD_EXIT
:FTP_UPLOAD_USAGE
echo Usage: ftpget [address] [username] [password] [local directory] [remote directory] [remote file pattern]
echo.
:FTP_UPLOAD_EXIT
set FTPADDRESS=
set FTPUSERNAME=
set FTPPASSWORD=
set LOCALDIR=
set REMOTEFILE=
set REMOTEDIR=
set TMPFILE=
set CURRENT_DIRECTORY=
set RELATIVE_DIRECTORY=
@echo on