Feature #462: url support

- curl enhancements (push/pull multiple files)
- now supports push/pull to/from filesystem

src/Transport.cpp

@@ -95,7 +95,7 @@ int Transport::execute()
     argv[1] = opt;                    // -c
     argv[2] = (char*)cmdline.c_str();	// e.g. scp undo.data user@host:.task/
     argv[3] = NULL;                   // required by execv
-
+    
     int ret = execvp("sh", argv);
     delete[] argv;
 
@@ -116,4 +116,19 @@ int Transport::execute()
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+bool Transport::is_directory(const std::string& path)
+{
+  return path[path.length()-1] == '/';
+}
+
+
+////////////////////////////////////////////////////////////////////////////////
+bool Transport::is_filelist(const std::string& path)
+{
+  return (path.find ("*") != std::string::npos)
+      || (path.find ("?") != std::string::npos)
+      || (path.find ("{") != std::string::npos);
+}
+
+////////////////////////////////////////////////////////////////////////////////
 

src/Transport.h

@@ -40,6 +40,9 @@ public:
 	
   virtual void send (const std::string&) = 0;
   virtual void recv (std::string) = 0;
+  
+  static bool is_directory(const std::string&);
+  static bool is_filelist(const std::string&);
 
 protected:
   std::string executable;

src/TransportCurl.cpp

@@ -26,6 +26,8 @@
 ////////////////////////////////////////////////////////////////////////////////
 
 #include "TransportCurl.h"
+#include "text.h"
+#include "util.h"
 
 ////////////////////////////////////////////////////////////////////////////////
 TransportCurl::TransportCurl(const Uri& uri) : Transport(uri)
@@ -38,19 +40,44 @@ void TransportCurl::send(const std::string& source)
 {
 	if (uri.host == "") {
 		throw std::string ("Hostname is empty");
-	}
+	}    
-
+      
-	// Wildcards arent supported
+	if (is_filelist(source))
-	if ( (source.find ("*") != std::string::npos)
+  {
-		|| (source.find ("?") != std::string::npos) )
+    std::string::size_type pos;
-	{
+    pos = source.find("{");
-		throw std::string ("Failed to use curl with wildcards!");
+    
-	}
+    if (pos == std::string::npos)
+      throw std::string ("Curl does not support wildcards!");
+      
+    if (!uri.is_directory())
+      throw std::string ("'" + uri.path + "' is not a directory!");
+      
+    std::string toSplit;
+    std::string suffix;
+    std::string prefix;
+    std::vector<std::string> splitted;
+    
+    prefix  = source.substr (0, pos);
+    toSplit = source.substr (pos+1); 
+    
+    pos     = toSplit.find ("}");
+    suffix  = toSplit.substr (pos+1);
+    split (splitted, toSplit.substr(0, pos), ',');
+    
+    foreach (file, splitted)
+    {
+      arguments.push_back ("-T");
+      arguments.push_back (prefix + *file + suffix);
+    }
+  }
+  else
+  {
+    arguments.push_back ("-T");
+    arguments.push_back (source);
+  }
 
 	// cmd line is: curl -T source protocol://host:port/path
-	arguments.push_back ("-T");
-	arguments.push_back (source);
-
 	if (uri.port != "")
 	{
 		arguments.push_back (uri.protocol + "://" + uri.host + ":" + uri.port + "/" + uri.path);
@@ -70,13 +97,37 @@ void TransportCurl::recv(std::string target)
 	if (uri.host == "") {
 		throw std::string ("Hostname is empty");
 	}
-
+  
-	// Wildcards arent supported
+  if (is_filelist(uri.path))
-	if ( (uri.path.find ("*") != std::string::npos)
+  {
-		|| (uri.path.find ("?") != std::string::npos) )
+    std::string::size_type pos;
-	{
+    pos = uri.path.find("{");
-		throw std::string ("Failed to use curl with wildcards!");
+    
-	}
+    if (pos == std::string::npos)
+      throw std::string ("Curl does not support wildcards!");
+      
+    if (!is_directory(target))
+      throw std::string ("'" + target + "' is not a directory!");
+      
+    std::string toSplit;
+    std::string suffix;
+    std::string prefix = target;
+    std::vector<std::string> splitted;
+    toSplit = uri.path.substr (pos+1);
+    pos = toSplit.find ("}");
+    suffix = toSplit.substr (pos+1);
+    split (splitted, toSplit.substr(0, pos), ',');
+    
+    target = "";
+    foreach (file, splitted)
+    {
+      target += " -o " + prefix + *file + suffix;
+    }
+  }
+  else
+  {
+    target = "-o " + target;
+  }
 	
 	// cmd line is: curl protocol://host:port/path/to/source/file -o path/to/target/file
 	if (uri.port != "")
@@ -87,8 +138,7 @@ void TransportCurl::recv(std::string target)
 	{
 		arguments.push_back (uri.protocol + "://" + uri.host + "/" + uri.path);
 	}
-	
+  
-	arguments.push_back ("-o");
 	arguments.push_back (target);
 
 	if (execute())

src/TransportRSYNC.cpp

@@ -42,18 +42,8 @@ void TransportRSYNC::send(const std::string& source)
 
 	// Is there more than one file to transfer?
 	// Then path has to end with a '/'
-	if ( (source.find ("*") != std::string::npos)
+	if (is_filelist(source) && !uri.is_directory())
-		|| (source.find ("?") != std::string::npos)
+    throw std::string ("'" + uri.path + "' is not a directory!");
-		|| (source.find (" ") != std::string::npos) )
-	{
-		std::string::size_type pos;
-
-		pos = uri.path.find_last_of ("/");
-		if (pos != uri.path.length()-1)
-		{
-			uri.path = uri.path.substr (0, pos+1);
-		}
-	}
 
 	// cmd line is: rsync [--port=PORT] source [user@]host::path
 	if (uri.port != "")
@@ -85,16 +75,8 @@ void TransportRSYNC::recv(std::string target)
 
 	// Is there more than one file to transfer?
 	// Then target has to end with a '/'
-	if ( (uri.path.find ("*") != std::string::npos)
+	if (is_filelist(uri.path) && !is_directory(target))
-		|| (uri.path.find ("?") != std::string::npos) )
+    throw std::string ("'" + target + "' is not a directory!");
-	{
-		std::string::size_type pos;
-		pos = target.find_last_of ("/");
-		if (pos != target.length()-1)
-		{
-			target = target.substr( 0, pos+1);
-		}
-	}
 	
 	// cmd line is: rsync [--port=PORT] [user@]host::path target
 	if (uri.port != "")

src/TransportSSH.cpp

@@ -42,18 +42,8 @@ void TransportSSH::send(const std::string& source)
 
 	// Is there more than one file to transfer?
 	// Then path has to end with a '/'
-	if ( (source.find ("*") != std::string::npos)
+	if (is_filelist(source) && !uri.is_directory())
-		|| (source.find ("?") != std::string::npos)
+    throw std::string ("'" + uri.path + "' is not a directory!");
-		|| (source.find (" ") != std::string::npos) )
-	{
-		std::string::size_type pos;
-
-		pos = uri.path.find_last_of ("/");
-		if (pos != uri.path.length()-1)
-		{
-			uri.path = uri.path.substr (0, pos+1);
-		}
-	}
 
 	// cmd line is: scp [-p port] [user@]host:path
 	if (uri.port != "")
@@ -86,16 +76,8 @@ void TransportSSH::recv(std::string target)
 
 	// Is there more than one file to transfer?
 	// Then target has to end with a '/'
-	if ( (uri.path.find ("*") != std::string::npos)
+	if (is_filelist(uri.path) && !is_directory(target))
-		|| (uri.path.find ("?") != std::string::npos) )
+    throw std::string ("'" + target + "' is not a directory!");
-	{
-		std::string::size_type pos;
-		pos = target.find_last_of ("/");
-		if (pos != target.length()-1)
-		{
-			target = target.substr( 0, pos+1);
-		}
-	}
 
 	// cmd line is: scp [-p port] [user@]host:path
 	if (uri.port != "")

src/Uri.cpp

@@ -104,7 +104,7 @@ std::string Uri::parent () const
   {
     std::string::size_type slash = path.rfind ('/');
     if (slash != std::string::npos)
-      return path.substr (0, slash);
+      return path.substr (0, slash+1);
   }
 
   return "";

src/command.cpp

@@ -658,18 +658,27 @@ void handlePush (std::string& outs)
 
 			Transport* transport;
 			if ((transport = Transport::getTransport (uri)) != NULL )
-			{
+			{        
-        // TODO specify data files
+				transport->send (location.data + "/{pending,undo,completed}.data");
-				transport->send (location.data + "/*.data");
 				delete transport;
 			}
 			else
 			{
-        // TODO copy files
+        // copy files locally
-        //std::ifstream ifile (location.data + "/undo.data", std::ios_base::binary);
+        if (!uri.is_directory())
-        //std::ofstream ofile (location.data + "/undo.data", std::ios_base::binary);
+          throw std::string ("'" + uri.path + "' is not a directory!");
         
-				throw std::string ("Push to local targets is not (yet) supported.");
+        std::ifstream ifile1 ((location.data + "/undo.data").c_str(), std::ios_base::binary);
+        std::ofstream ofile1 ((uri.path      +  "undo.data").c_str(), std::ios_base::binary);        
+        ofile1 << ifile1.rdbuf();
+        
+        std::ifstream ifile2 ((location.data + "/pending.data").c_str(), std::ios_base::binary);
+        std::ofstream ofile2 ((uri.path      +  "pending.data").c_str(), std::ios_base::binary);        
+        ofile2 << ifile2.rdbuf();
+        
+        std::ifstream ifile3 ((location.data + "/completed.data").c_str(), std::ios_base::binary);
+        std::ofstream ofile3 ((uri.path      +  "completed.data").c_str(), std::ios_base::binary);        
+        ofile3 << ifile3.rdbuf();
 			}
 
       context.hooks.trigger ("post-push-command");
@@ -693,7 +702,8 @@ void handlePull (std::string& outs)
     {
 			Directory location (context.config.get ("data.location"));
 
-			uri.append ("*.data");
+			if (!uri.append ("{pending,undo,completed}.data"))
+        throw std::string ("'" + uri.path + "' is not a directory!");
 
 			Transport* transport;
 			if ((transport = Transport::getTransport (uri)) != NULL )
@@ -702,9 +712,34 @@ void handlePull (std::string& outs)
 				delete transport;
 			}
 			else
-			{
+			{        
-        // TODO copy files
+        // copy files locally
-				throw std::string ("Pull failed");
+        
+        // remove {pending,undo,completed}.data
+        uri.path = uri.parent();     
+        
+        Path path1 (uri.path + "undo.data");
+        Path path2 (uri.path + "pending.data");
+        Path path3 (uri.path + "completed.data");
+        
+        if (path1.exists() && path2.exists() && path3.exists())
+        {
+          std::ofstream ofile1 ((location.data + "/undo.data").c_str(), std::ios_base::binary);
+          std::ifstream ifile1 (path1.data.c_str()                    , std::ios_base::binary);        
+          ofile1 << ifile1.rdbuf();
+          
+          std::ofstream ofile2 ((location.data + "/pending.data").c_str(), std::ios_base::binary);
+          std::ifstream ifile2 (path2.data.c_str()                    , std::ios_base::binary);            
+          ofile2 << ifile2.rdbuf();
+          
+          std::ofstream ofile3 ((location.data + "/completed.data").c_str(), std::ios_base::binary);
+          std::ifstream ifile3 (path3.data.c_str()                    , std::ios_base::binary);          
+          ofile3 << ifile3.rdbuf();
+        }
+        else
+        {
+          throw std::string ("At least one of the database files in '" + uri.path + "' is not present.");
+        }       
 			}
 
       context.hooks.trigger ("post-pull-command");