"Fossies" - the Fresh Open Source Software Archive

Member "sitecopy-0.16.6/src/sites.c" (9 Jul 2008, 36732 Bytes) of archive /linux/www/sitecopy-0.16.6.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "sites.c" see the Fossies "Dox" file reference documentation.

    1 /* 
    2    sitecopy, for managing remote web sites.
    3    Copyright (C) 1998-2008, Joe Orton <joe@manyfish.co.uk>
    4                                                                      
    5    This program is free software; you can redistribute it and/or modify
    6    it under the terms of the GNU General Public License as published by
    7    the Free Software Foundation; either version 2 of the License, or
    8    (at your option) any later version.
    9   
   10    This program is distributed in the hope that it will be useful,
   11    but WITHOUT ANY WARRANTY; without even the implied warranty of
   12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   13    GNU General Public License for more details.
   14   
   15    You should have received a copy of the GNU General Public License
   16    along with this program; if not, write to the Free Software
   17    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
   18 
   19 */
   20 
   21 /* This is the core functionality of sitecopy, performing updates
   22  * and checking files etc. */
   23 
   24 #include <config.h>
   25 
   26 #include <sys/types.h>
   27 #include <sys/stat.h>
   28 
   29 #include <errno.h>
   30 #include <dirent.h>
   31 #include <fnmatch.h>
   32 #include <fcntl.h>
   33 #include <stdio.h>
   34 #ifdef HAVE_STDLIB_H
   35 #include <stdlib.h>
   36 #endif /* HAVE_STDLIB_H */
   37 #ifdef HAVE_UNISTD_H
   38 #include <unistd.h>
   39 #endif /* HAVE_UNISTD_H */
   40 #ifdef HAVE_STRING_H
   41 #include <string.h>
   42 #endif
   43 #ifdef HAVE_STRINGS_H
   44 #include <strings.h>
   45 #endif
   46 #include <time.h>
   47 #include <utime.h>
   48 
   49 /* neon */
   50 #include <ne_string.h>
   51 #include <ne_alloc.h>
   52 #include <ne_md5.h>
   53 #include <ne_socket.h>
   54 
   55 #ifdef HAVE_SNPRINTF_H
   56 #include "snprintf.h"
   57 #endif /* !HAVE_SNPRINTF_H */
   58 
   59 #include "basename.h"
   60 
   61 #include "i18n.h"
   62 #include "common.h"
   63 #include "frontend.h"
   64 #include "protocol.h"
   65 #include "sitesi.h"
   66 
   67 /* Shorthand for protocol driver methods */
   68 #define CALL(a) (*site->driver->a)
   69 #define DRIVER_ERR  ((*site->driver->error)(session))
   70 
   71 /* This holds ALL the sites defined in the rcfile */
   72 struct site *all_sites;
   73  
   74 static int proto_init(struct site *site, void **session);
   75 static void proto_finish(struct site *site, void *session);
   76 static void proto_seterror(struct site *site, void *session);
   77 
   78 struct site *site_find(const char *sitename)
   79 {
   80     struct site *current;
   81 
   82     for (current = all_sites; current!=NULL; current=current->next) {
   83     if (strcmp(current->name, sitename) == 0) {
   84         /* We found it */
   85         return current;
   86     }
   87     }
   88 
   89     return NULL;
   90 }
   91 
   92 static int synch_create_directories(struct site *site)
   93 {
   94     struct site_file *current;
   95     char *full_local;
   96     int ret;
   97     
   98     ret = 0;
   99     
  100     for_each_file(current, site) {
  101     if ((current->type==file_dir) && (current->diff==file_deleted)) {
  102         full_local = file_full_local(&current->stored, site);
  103         fe_synching(current);
  104         if (mkdir(full_local, 0755) == 0) {
  105         fe_synched(current, true, NULL);
  106         } else {
  107         ret = 1;
  108         fe_synched(current, false, strerror(errno));
  109         file_downloaded(current, site);
  110         }
  111         free(full_local);
  112     }
  113     }
  114     return ret;
  115 }
  116 
  117 static int synch_files(struct site *site, void *session)
  118 {
  119     struct site_file *current;
  120     int ret;
  121 
  122     ret = 0;
  123 
  124     for_each_file(current, site) {
  125     char *full_local, *full_remote;
  126     if (current->type != file_file) continue;
  127     switch (current->diff) {
  128     case file_changed:
  129         if (!file_contents_changed(current, site)) {
  130         /* Just chmod it */
  131         full_local = file_full_local(&current->stored, site);
  132         fe_setting_perms(current);
  133         if (chmod(full_local, current->stored.mode) < 0) {
  134             fe_set_perms(current, false, strerror(errno));
  135         } else {
  136             fe_set_perms(current, true, NULL);
  137         }
  138         free(full_local);
  139         break;
  140         }
  141         /*** fall-through */
  142     case file_deleted:
  143         full_local = file_full_local(&current->stored, site);
  144         full_remote = file_full_remote(&current->stored, site);
  145         fe_synching(current);
  146         if (CALL(file_download)(session, full_local, full_remote,
  147                     current->stored.ascii) != SITE_OK) {
  148         fe_synched(current, false, DRIVER_ERR);
  149         ret = 1;
  150         } else { 
  151         /* Successfull download */
  152         fe_synched(current, true, NULL);
  153         if (site->state_method == state_timesize) {
  154             struct utimbuf times;
  155             /* Change the modtime of the local file so it doesn't look
  156              * like it's changed already */
  157             times.actime = current->stored.time;
  158             times.modtime = current->stored.time;
  159             if (utime(full_local, &times) < 0) {
  160             fe_warning(_("Could not set modification time of local file."),
  161                     full_local, strerror(errno));
  162             }
  163         }
  164         if (file_perms_changed(current, site)) {
  165             fe_setting_perms(current);
  166             if (chmod(full_local, current->stored.mode) < 0) {
  167             fe_set_perms(current, false, strerror(errno));
  168             } else {
  169             fe_set_perms(current, true, NULL);
  170             }
  171         }
  172         /* TODO: not strictly true if the chmod failed. */
  173         file_downloaded(current, site);
  174         }
  175         free(full_local);
  176         free(full_remote);
  177         break;
  178     case file_new:
  179         full_local = file_full_local(&current->local, site);
  180         fe_synching(current);
  181         if (unlink(full_local) != 0) {
  182         fe_synched(current, false, strerror(errno));
  183         ret = 1;
  184         } else {
  185         fe_synched(current, true, NULL);
  186         }
  187         free(full_local);
  188         break;
  189     case file_moved: {
  190         char *old_full_local = file_full_local(&current->stored, site);
  191         full_local = file_full_local(&current->local, site);
  192         fe_synching(current);
  193         if (rename(full_local, old_full_local) == 0) {
  194         fe_synched(current, true, NULL);
  195         } else {
  196         fe_synched(current, false, strerror(errno));
  197         ret = 1;
  198         }
  199         free(old_full_local);
  200         free(full_local);
  201     }
  202     default:
  203         break;      
  204     }
  205     }
  206 
  207     return ret;
  208 }
  209 
  210 static int synch_delete_directories(struct site *site)
  211 {
  212     struct site_file *current, *prev;
  213     int ret;
  214 
  215     ret = 0;
  216 
  217     for (current=site->files_tail; current!=NULL; current=prev) {
  218     prev = current->prev;
  219     if ((current->type==file_dir) && (current->diff==file_new)) {
  220         char *full_local = file_full_local(&current->local, site);
  221         fe_synching(current);
  222         if (rmdir(full_local) == -1) {
  223         fe_synched(current, false, strerror(errno));
  224         ret = 3;
  225         } else {
  226         fe_synched(current, true, NULL);
  227         file_delete(site, current);
  228         }
  229         free(full_local);
  230     }
  231     }
  232 
  233     return ret;
  234 }
  235 
  236 /* Resyncs the LOCAL site with the REMOTE site.
  237  * This is site_update backwards, and is essentially the same in structure,
  238  * except with the logic reversed.
  239  */
  240 int site_synch(struct site *site) 
  241 {
  242     int ret, need_conn;
  243     void *session;
  244  
  245     /* Do we need to connect to the server: note that ignored files
  246      * are treated as changed files in synch mode. */
  247     need_conn = (site->numchanged + site->numdeleted + 
  248          site->numignored > 0);
  249    
  250     if (need_conn) {
  251     ret = proto_init(site, &session);
  252     if (ret != SITE_OK) {
  253         proto_finish(site, session);
  254         return ret;
  255     }
  256     }
  257 
  258     ret = synch_create_directories(site);
  259     if (ret == 0 || site->keep_going) {
  260     ret = synch_files(site, session);
  261     if (ret == 0 || site->keep_going) {
  262         ret = synch_delete_directories(site);
  263     }
  264     }
  265     
  266     if (need_conn) {
  267     proto_finish(site, session);
  268     }
  269 
  270     if (ret == 0) {
  271     ret = SITE_OK;
  272     } else {
  273     ret = SITE_ERRORS;
  274     }
  275     return ret;
  276 }
  277 
  278 static int file_chmod(struct site_file *file, struct site *site, void *session)
  279 {
  280     int ret = 0;
  281     /* chmod it if necessary */
  282     if (file_perms_changed(file, site)) {
  283     char *full_remote = file_full_remote(&file->local, site);
  284     fe_setting_perms(file);
  285     if (CALL(file_chmod)(session, full_remote, file->local.mode) != SITE_OK) {
  286         fe_set_perms(file, false, DRIVER_ERR);
  287         ret = 1;
  288     } else {
  289         file->stored.mode = file->local.mode;
  290         fe_set_perms(file, true, NULL);
  291         file_set_diff(file, site);
  292     }
  293     free(full_remote);
  294     }
  295     return ret;
  296 }
  297 
  298 static void 
  299 file_retrieve_server(struct site_file *file, struct site *site, void *session)
  300 {
  301     time_t rtime;
  302     char *full_remote = file_full_remote(&file->local, site);
  303     if (CALL(file_get_modtime)(session, full_remote, &rtime) == SITE_OK) {
  304     file->server.time = rtime;
  305     file->server.exists = true;
  306     } else {
  307     file->server.exists = false;
  308     fe_warning(_("Upload succeeded, but could not retrieve modification time.\n"
  309               "If this message persists, turn off safe mode."),
  310             full_remote, DRIVER_ERR);
  311     }
  312     free(full_remote);
  313 }
  314 
  315 /* Create new directories and change permissions on existing directories. */
  316 static int update_create_directories(struct site *site, void *session)
  317 {
  318     struct site_file *current;
  319     int ret = 0;
  320 
  321     for_each_file(current, site) {
  322     if ((current->type == file_dir) 
  323             && (current->diff == file_new || current->diff == file_changed)) {
  324         /* New or changed directory! */
  325         char *full_remote;
  326             int oret;
  327 
  328         if (!fe_can_update(current)) continue;
  329 
  330         full_remote = file_full_remote(&current->local, site);
  331 
  332             if (current->diff == file_new) {
  333                 fe_updating(current);
  334                 oret = CALL(dir_create)(session, full_remote);
  335                 if (oret != SITE_OK) {
  336                     fe_updated(current, false, DRIVER_ERR);
  337                 } else {
  338                     fe_updated(current, true, NULL);
  339                 }
  340             } else {
  341                 oret = SITE_OK;
  342             }
  343 
  344             if (site->dirperms && oret == SITE_OK) {
  345                 fe_setting_perms(current);
  346                 oret = CALL(file_chmod)(session, full_remote,
  347                                         current->local.mode);
  348                 if (oret == SITE_OK) {
  349                     fe_set_perms(current, true, NULL);
  350                 } else {
  351                     fe_set_perms(current, false, DRIVER_ERR);
  352                 }
  353             }
  354 
  355             if (oret != SITE_OK) {
  356                 ret = 1;
  357             } else {
  358                 file_uploaded(current, site);
  359             }
  360         free(full_remote);
  361     }
  362     }
  363 
  364     return ret;
  365 }
  366 
  367 /* Returns the filename to use for tempupload mode, ne_malloc-allocated.
  368  * (pass the site since we may have different tempupload modes in the
  369  * future.)
  370  * FIXME: implement it efficiently */
  371 static char *temp_upload_filename(const char *filename, struct site *site)
  372 {
  373     char *pnt, *ret;
  374     /* Insert a '.in.' prefix into the filename, AFTER
  375      * any directories */
  376     ret = ne_malloc(strlen(filename) + 4 + 1);
  377     strcpy(ret, filename);
  378     pnt = strrchr(ret, '/');
  379     if (pnt == NULL) {
  380     pnt = ret;
  381     } else {
  382     pnt++;
  383     }
  384     /* Shove the name segment along four bytes so we can insert
  385      * the '.in.' */
  386     memmove(pnt+4, pnt, strlen(pnt) + 1);
  387     memcpy(pnt, ".in.", 4);
  388     return ret;
  389 }
  390         
  391 static int update_delete_files(struct site *site, void *session)
  392 {
  393     struct site_file *current, *next;
  394     int ret = 0;
  395 
  396     for (current=site->files; current!=NULL; current=next) {
  397     next = current->next;
  398     /* Skip directories and links, and only do deleted files on
  399      * this pass */
  400     if (current->diff == file_deleted &&
  401         current->type == file_file) {
  402         char *full_remote;
  403         if (!fe_can_update(current)) continue;
  404         full_remote = file_full_remote(&current->stored, site);
  405         fe_updating(current);
  406         if (CALL(file_delete)(session, full_remote) != SITE_OK) {
  407         fe_updated(current, false, DRIVER_ERR);
  408         ret = 1;
  409         } else {
  410         /* Successful update - file was deleted */
  411         fe_updated(current, true, NULL);
  412         file_delete(site, current);
  413         }
  414         free(full_remote);
  415     }
  416     }
  417     
  418     return ret;
  419 }
  420 
  421 static int update_move_files(struct site *site, void *session)
  422 {
  423     int ret = 0;
  424     struct site_file *current;
  425     char *old_full_remote, *full_remote;
  426     for_each_file(current, site) {
  427     if (current->diff != file_moved) 
  428         continue;
  429     full_remote = file_full_remote(&current->local, site);
  430     /* The file has been moved */
  431     if (!fe_can_update(current)) continue;
  432     fe_updating(current);
  433     old_full_remote = file_full_remote(&current->stored, site);
  434     if (CALL(file_move)(session, old_full_remote, full_remote) != SITE_OK) {
  435         ret = 1;
  436         fe_updated(current, false, DRIVER_ERR);
  437     } else {
  438         /* Successful update - file was moved */
  439         fe_updated(current, true, NULL);
  440         file_uploaded(current, site);
  441     }
  442     free(old_full_remote);
  443     free(full_remote);
  444     }   
  445 
  446     return ret;
  447 }
  448 
  449 
  450 /* Does everything but file deletes */
  451 static int update_files(struct site *site, void *session)
  452 {
  453     struct site_file *current;
  454     char *full_local, *full_remote;
  455     int ret = 0;
  456 
  457     for_each_file(current, site) {
  458 
  459     /* This loop only handles changed and new files, so
  460      * skip everything else. */
  461 
  462     if (current->type != file_file
  463         || current->diff == file_deleted
  464         || current->diff == file_moved
  465         || current->diff == file_unchanged) continue;
  466 
  467     full_local = file_full_local(&current->local, site);
  468     full_remote = file_full_remote(&current->local, site);
  469 
  470     switch (current->diff) {
  471     case file_changed: /* File has changed, upload it */
  472         if (current->ignore) break;
  473         if (!file_contents_changed(current, site)) {
  474         /* If the file contents haven't changed, then we can
  475          * just chmod it */
  476         if (file_chmod(current, site, session))
  477             ret = 1;
  478         break;
  479         }
  480         /*** fall-through ***/
  481     case file_new: /* File is new, upload it */
  482         if (!fe_can_update(current)) continue;
  483         if ((current->diff == file_changed) && site->nooverwrite) {
  484         /* Must delete remote file before uploading new copy.
  485          * FIXME: Icky hack to convince the FE we are about to
  486          * delete the file */
  487         current->diff = file_deleted;
  488         fe_updating(current);
  489         if (CALL(file_delete)(session, full_remote) != SITE_OK) {
  490             fe_updated(current, false, DRIVER_ERR);
  491             ret = 1;
  492             current->diff = file_changed;
  493             /* Don't upload it! */
  494             break;
  495         } else {
  496             fe_updated(current, true, NULL);
  497             current->diff = file_changed;
  498         }
  499         }
  500         fe_updating(current);
  501         /* Now, upload it */
  502         if (site->safemode && current->server.exists) {
  503         /* Only do this for files we do know the remote modtime for */
  504         int cret;
  505         cret = CALL(file_upload_cond)(session,
  506             full_local, full_remote, current->local.ascii,
  507             current->server.time);
  508         switch (cret) {
  509         case SITE_ERRORS:
  510             fe_updated(current, false, DRIVER_ERR);
  511             ret = 1;
  512             break;
  513         case SITE_FAILED:
  514             fe_updated(current, false, 
  515                 _("Remote file has been modified - not overwriting with local changes"));
  516             ret = 1;
  517             break;
  518         default:
  519             /* Success case */
  520             fe_updated(current, true, NULL);
  521             file_retrieve_server(current, site, session);
  522             if (file_chmod(current, site, session)) ret = 1;
  523             file_uploaded(current, site);
  524             break;
  525         }
  526         } else if (site->tempupload) {
  527         /* Do temp file upload followed by a move */
  528         char *temp_remote = temp_upload_filename(full_remote, site);
  529         if (CALL(file_upload)(session, full_local, temp_remote,
  530                        current->local.ascii != SITE_OK)) {
  531             fe_updated(current, false, DRIVER_ERR);
  532             ret = 1;
  533         } else {
  534             /* Successful upload... now move it */
  535             if (CALL(file_move)(session, temp_remote, 
  536                      full_remote) != SITE_OK) {
  537             fe_updated(current, false, DRIVER_ERR);
  538             /* Originally coded to delete the temporary file
  539              * here, but, on second thoughts... if something
  540              * is broken, let's not try to be too clever, else
  541              * we might make it worse. */
  542             ret = 1;
  543             } else {
  544             /* Successful move */
  545             fe_updated(current, true, NULL);
  546             if (site->safemode) {
  547                 file_retrieve_server(current, site, session);
  548             }
  549             if (file_chmod(current, site, session)) ret = 1;
  550             file_uploaded(current, site);
  551             }
  552         }
  553         free(temp_remote);
  554         } else {
  555         /* Normal unconditional upload */
  556         if (CALL(file_upload)(session, full_local, full_remote, 
  557                        current->local.ascii) != SITE_OK) {
  558             fe_updated(current, false, DRIVER_ERR);
  559             ret = 1;
  560         } else {
  561             /* Successful upload. */
  562             fe_updated(current, true, NULL);
  563             if (site->safemode) {
  564             file_retrieve_server(current, site, session);
  565             }
  566             if (file_chmod(current, site, session)) ret = 1;
  567             file_uploaded(current, site);
  568         }
  569         }
  570         break;
  571         
  572     default: /* Ignore everything else */
  573         break;
  574     }
  575     free(full_remote);
  576     free(full_local);
  577     }
  578 
  579     return ret;
  580     
  581 }
  582 
  583 static int update_delete_directories(struct site *site, void *session)
  584 {
  585     struct site_file *current, *prev;
  586     int ret = 0;
  587 
  588     /* This one must iterate through the list BACKWARDS, so
  589      * directories are deleted bottom up */
  590     for (current=site->files_tail; current!=NULL; current=prev) {
  591     prev = current->prev;
  592     if ((current->type==file_dir) && (current->diff == file_deleted)) {
  593         char *full_remote;
  594         if (!fe_can_update(current)) continue;
  595         full_remote = file_full_remote(&current->stored, site);
  596         fe_updating(current);
  597         if (CALL(dir_remove)(session, full_remote) != SITE_OK) {
  598         ret = 1;
  599         fe_updated(current, false, DRIVER_ERR);
  600         } else {
  601         /* Successful delete */
  602         fe_updated(current, true, NULL);
  603         file_delete(site, current);
  604         }
  605         free(full_remote);
  606     }
  607     }
  608     return ret;
  609 }
  610 
  611 static int update_links(struct site *site, void *session)
  612 {
  613     struct site_file *current, *next;
  614     int ret = 0;
  615 
  616     for (current=site->files; current!=NULL; current=next) {
  617     char *full_remote;
  618     next = current->next;
  619     if (current->type != file_link) continue;
  620 
  621     full_remote = file_full_remote(&current->local, site);
  622     switch (current->diff) {
  623     case file_new:
  624         fe_updating(current);
  625         if (CALL(link_create)(session, full_remote, 
  626                   current->local.linktarget) != SITE_OK) {
  627         fe_updated(current, false, DRIVER_ERR);
  628         ret = 1;
  629         } else {
  630         fe_updated(current, true, NULL);
  631         current->diff = file_unchanged;
  632         }
  633         break;
  634     case file_changed:
  635         fe_updating(current);
  636         if (CALL(link_change)(session, full_remote,
  637                    current->local.linktarget) != SITE_OK) {
  638         fe_updated(current, false, DRIVER_ERR);
  639         ret = 1;
  640         } else {
  641         fe_updated(current, true, NULL);
  642         current->diff = file_unchanged;
  643         }
  644         break;
  645     case file_deleted:
  646         fe_updating(current);
  647         if (CALL(link_delete)(session, full_remote) != SITE_OK) {
  648         fe_updated(current, false, DRIVER_ERR);
  649         ret = 1;
  650         } else {
  651         fe_updated(current, true, NULL);
  652         file_delete(site, current);
  653         }
  654     default:
  655         break;
  656     }
  657     free(full_remote);
  658     }
  659     return ret;
  660 }
  661 
  662 static void proto_finish(struct site *site, void *session)
  663 {
  664     proto_seterror(site, session);
  665     CALL(finish)(session);
  666 }
  667 
  668 static void proto_seterror(struct site *site, void *session)
  669 {
  670     site->last_error = ne_strdup(DRIVER_ERR);
  671 }
  672 
  673 const char *site_get_protoname(struct site *site) 
  674 {
  675     if (site->driver)
  676     return site->driver->protocol_name;
  677     else
  678     return site->proto_string;
  679 }
  680 
  681 static int proto_init(struct site *site, void **session)
  682 {
  683     int ret;
  684     
  685     if (site->last_error) {
  686     free(site->last_error);
  687     site->last_error = NULL;
  688     }
  689 
  690     ret = CALL(init)(session, site);
  691     if (ret != SITE_OK) {
  692     proto_seterror(site, *session);
  693     return ret;
  694     }
  695 
  696     return SITE_OK;
  697 }
  698 
  699 
  700 /* Updates the remote site.
  701  * 
  702  * Executes each of the site_update_* functions in turn (if their
  703  * guard evaluates to true). 
  704  */
  705 int site_update(struct site *site)
  706 {
  707     int ret = 0, num;
  708     const struct handler {
  709     int (*func)(struct site *, void *session);
  710     int guard;
  711     } handlers[] = {
  712     { update_delete_files, !site->nodelete },
  713     { update_create_directories, 1 },
  714     { update_move_files, site->checkmoved },
  715     { update_files, 1 },
  716     { update_links, site->symlinks == sitesym_maintain },
  717     { update_delete_directories, !site->nodelete },
  718     { NULL, 1 }
  719     };
  720     void *session;
  721 
  722     ret = proto_init(site, &session);
  723     if (ret != SITE_OK) {
  724     proto_finish(site, session);
  725     return ret;
  726     }
  727     
  728     for (num = 0; handlers[num].func != NULL && (ret == 0 || site->keep_going);
  729      num++) {
  730     if (handlers[num].guard) {
  731         int newret;
  732         newret = (*handlers[num].func)(site, session);
  733         if (newret != 0) {
  734         ret = newret;
  735         }
  736     }
  737     }
  738 
  739     if (ret == 0) {
  740     /* Site updated successfully. */
  741     ret = SITE_OK;
  742     } else {
  743     /* Update not totally successfull */
  744     ret = SITE_ERRORS;
  745     }
  746     
  747     proto_finish(site, session);
  748 
  749     return ret;
  750 }
  751 
  752 /* This reads off the remote files and the local files. */
  753 int site_readfiles(struct site *site)
  754 {
  755     int ret;
  756     site_destroy(site);
  757     ret = site_read_stored_state(site);
  758     if (ret == SITE_OK) {
  759     site_read_local_state(site);
  760     }
  761     return ret;
  762 }
  763 
  764 /* Read the local site files... 
  765  * A stack is used for directories within the site - this is not recursive.
  766  * Each item on the stack is a FULL PATH to the directory, i.e., including
  767  * the local site root. */
  768 
  769 /* Initial size of directory stack, and amount it grows
  770  * each time we fill it. */
  771 #define DIRSTACKSIZE (1024)
  772 
  773 void site_read_local_state(struct site *site)
  774 {
  775     char **dirstack, *this, *full = NULL;
  776     int dirtop = 0, /* points to item above top stack item */
  777     dirmax = DIRSTACKSIZE; /* size of stack */
  778 
  779     dirstack = ne_malloc(sizeof(char *) * DIRSTACKSIZE);
  780     /* Push the root directory on to the stack */
  781     dirstack[dirtop++] = ne_strdup(site->local_root);
  782     
  783     /* Now, for all items in the stack, process all the files, and
  784      * add the dirs to the stack. Everything we put on the stack is
  785      * temporary and gets freed eventually. */
  786 
  787     while (dirtop > 0) {
  788     DIR *curdir;
  789     struct dirent *ent;
  790     /* Pop the stack */
  791     this = dirstack[--dirtop];
  792     
  793     NE_DEBUG(DEBUG_FILES, "Scanning: %s\n", this);
  794     curdir = opendir(this);
  795     if (curdir == NULL) {
  796         fe_warning("Could not read directory", this, strerror(errno));
  797         free(this);
  798         continue;
  799     }
  800     
  801     /* Now read all the directory entries */
  802     while ((ent = readdir(curdir)) != NULL) {
  803         char *fname;
  804         struct stat item;
  805         struct site_file *current;
  806         struct file_state local = {0};
  807         enum file_type type;
  808         size_t dnlen = strlen(ent->d_name);
  809 
  810         /* Exclude the special directory entries. This test comes
  811          * high since it kills two stat calls per directory. */
  812         if (ent->d_name[0] == '.' && 
  813         (dnlen == 1 || (ent->d_name[1] == '.' && dnlen==2))) {
  814         continue;
  815         }
  816         
  817         if (full != NULL) free(full);
  818 
  819         full = ne_concat(this, ent->d_name, NULL);
  820 
  821 #ifdef __EMX__
  822 /* There are no symlinks under OS/2, use stat() instead */
  823 #define USE_STAT stat
  824 #else 
  825 #define USE_STAT lstat
  826 #endif
  827         if (USE_STAT(full, &item) == -1) {
  828         fe_warning(_("Could not examine file."), full, strerror(errno));
  829         continue;
  830         }
  831 #undef USE_STAT
  832 
  833 #ifndef __EMX__
  834         /* Is this a symlink? */
  835         if (S_ISLNK(item.st_mode)) {
  836         NE_DEBUG(DEBUG_FILES, "symlink - ");
  837         if (site->symlinks == sitesym_ignore) {
  838             /* Just skip it */
  839             NE_DEBUG(DEBUG_FILES, "ignoring.\n");
  840             continue;
  841         } else if (site->symlinks == sitesym_follow) {
  842             NE_DEBUG(DEBUG_FILES, "followed - ");
  843             /* Else, carry on as normal, stat the real file */
  844             if (stat(full, &item) == -1) {
  845             /* It's probably a broken link */
  846             NE_DEBUG(DEBUG_FILES, "broken.\n");
  847             continue;
  848             }
  849         } else {
  850             NE_DEBUG(DEBUG_FILES, "maintained:\n");
  851         }
  852         }
  853 #endif /* __EMX__ */
  854         /* Now process it */
  855         
  856         /* This is the filename of this file - i.e., everything
  857          * apart from the local root */
  858         fname = (char *)full+strlen(site->local_root);
  859         
  860         /* Check for excludes */
  861         if (file_isexcluded(fname, site))
  862         continue;
  863         
  864         if (S_ISREG(item.st_mode)) {
  865         switch (site->state_method) {
  866         case state_timesize:
  867             local.time = item.st_mtime;
  868             break;
  869         case state_checksum:
  870             if (file_checksum(full, &local, site) != 0) {
  871             fe_warning(_("Could not checksum file"), full,
  872                     strerror(errno));
  873             continue;
  874             }
  875             break;
  876         }
  877         local.size = item.st_size;
  878         local.ascii = file_isascii(fname, site);
  879         type = file_file;
  880         }
  881 #ifndef __EMX__
  882         else if (S_ISLNK(item.st_mode)) {
  883         char tmp[BUFSIZ] = {0};
  884         type = file_link;
  885         NE_DEBUG(DEBUG_FILES, "symlink being maintained.\n");
  886         if (readlink(full, tmp, BUFSIZ) == -1) {
  887             fe_warning(_("The target of the symlink could not be read."), full, strerror(errno));
  888             continue;
  889         }
  890         local.linktarget = ne_strdup(tmp);
  891         }
  892 #endif /* __EMX__ */
  893         else if (S_ISDIR(item.st_mode)) {
  894         type = file_dir;
  895         if (dirtop == dirmax) {
  896             /* Grow the stack */
  897             dirmax += DIRSTACKSIZE;
  898             dirstack = realloc(dirstack, sizeof(char *) * dirmax);
  899         }
  900         /* Add it to the search stack */
  901         dirstack[dirtop] = ne_concat(full, "/", NULL);
  902         dirtop++;
  903         } else {
  904         NE_DEBUG(DEBUG_FILES, "something else.\n");
  905         continue;
  906         }
  907         
  908         /* Set up rest of the local state */
  909         local.mode = item.st_mode & 0777;
  910         local.exists = true;
  911         local.filename = ne_strdup(fname);
  912 
  913         current = file_set_local(type, &local, site);
  914         DEBUG_DUMP_FILE_PROPS(DEBUG_FILES, current, site);
  915 
  916     }
  917     /* Close the open directory */
  918     closedir(curdir);
  919     /* And we're finished with this */
  920     free(this);
  921     }
  922 
  923     free(dirstack);
  924 }
  925 
  926 /* Pretend the remote site is the same as the local site. */
  927 void site_catchup(struct site *site)
  928 {
  929     struct site_file *current, *next;
  930     for (current=site->files; current!=NULL; current=next) {
  931     next = current->next;
  932     switch (current->diff) {
  933     case file_deleted:
  934         file_delete(site, current);
  935         break;
  936     case file_changed:
  937     case file_new:
  938     case file_moved:
  939         file_state_copy(&current->stored, &current->local, site);
  940         file_set_diff(current, site);
  941         break;
  942     case file_unchanged:
  943         /* noop */
  944         break;
  945     }
  946     }
  947 }
  948 
  949 /* Reinitializes the site - clears any remote files
  950  * from the list, and marks all other files as 'new locally'.
  951  */
  952 void site_initialize(struct site *site)
  953 {
  954     /* So simple. Be sure we have our abstraction layers at least
  955      * half-decent when things fall out this simple. */
  956     site_destroy_stored(site);
  957 }
  958 
  959 /* Munge modtimes of 'file' accordingly; when modtime of file on
  960  * server is 'remote_mtime'. */
  961 static void munge_modtime(struct site_file *file, time_t remote_mtime,
  962               struct site *site)
  963 {
  964     /* If this is a file, and we are using timesize mode, and we have
  965      * a local copy of this file already, we have to cope with the
  966      * modtimes problem.  The problem is that the modtime locally will
  967      * ALWAYS be different from the modtime on the SERVER.  */
  968     if (file->type == file_file && site->state_method == state_timesize) {
  969     if (file->local.exists) {
  970         /* If we are in safe mode, we can actually check whether
  971          * the remote file has changed or not when we are using
  972          * timesize mode, by comparing what we thought the server
  973          * modtime was with what the actual (fetched) server
  974          * modtime is. Got that? */
  975         NE_DEBUG(DEBUG_FILES, "Fetch: %ld vs %ld\n", 
  976              file->server.time, remote_mtime);
  977         if (site->safemode && file->server.exists &&
  978         file->server.time != remote_mtime) {
  979         NE_DEBUG(DEBUG_FILES, 
  980              "Fetch: Marking changed file changed.\n");
  981         file->stored.time = file->local.time + 1;
  982         } else {
  983         NE_DEBUG(DEBUG_FILES, "Fetch: Marking unchanged files same.\n");
  984         file->stored.time = file->local.time;
  985         }
  986     } else {
  987         /* If the local file doesn't exist, pretend the file was
  988          * last uploaded "now" (an arbitrary time is adequate, but
  989          * "now" is the least confusing). */
  990         file->stored.time = time(NULL);
  991     }
  992 
  993     /* update the diff. */
  994     file_set_diff(file, site);
  995     }
  996 }
  997 
  998 /* Return a site_file structure given a proto_file structure fetched
  999  * by the protocol driver. */
 1000 static struct site_file *fetch_add_file(struct site *site,
 1001                                         const struct proto_file *pf)
 1002 {
 1003     enum file_type type = file_file; /* init to shut up gcc */
 1004     struct site_file *file;
 1005     struct file_state state = {0};
 1006     
 1007     switch (pf->type) {
 1008     case proto_file:
 1009         type = file_file;
 1010         break;
 1011     case proto_dir:
 1012         type = file_dir;
 1013         break;
 1014     case proto_link:
 1015         type = file_link;
 1016         break;
 1017     }
 1018 
 1019     state.size = pf->size;
 1020     state.time = pf->modtime;
 1021     state.exists = true;
 1022     state.filename = pf->filename;
 1023     state.mode = pf->mode;
 1024     state.ascii = file_isascii(pf->filename, site);
 1025     memcpy(state.checksum, pf->checksum, 16);
 1026     
 1027     file = file_set_stored(type, &state, site);
 1028     
 1029     munge_modtime(file, pf->modtime, site);
 1030     
 1031     if (site->safemode) {
 1032         /* Store the server modtime. */
 1033         file->server.time = pf->modtime;
 1034         file->server.exists = true;
 1035     }
 1036 
 1037     return file;
 1038 }
 1039 
 1040 static
 1041 #if NE_VERSION_MINOR == 24
 1042 void
 1043 #else
 1044 int
 1045 #endif
 1046 site_fetch_csum_read(void *userdata, const char *s, size_t len)
 1047 {
 1048     struct ne_md5_ctx *md5 = userdata;
 1049     ne_md5_process_bytes(s, len, md5);
 1050 #if NE_VERSION_MINOR != 24
 1051     return 0;
 1052 #endif
 1053 }
 1054 
 1055 /* Retrieve the remote checksum for all files */
 1056 static int fetch_checksum_file(struct proto_file *file,
 1057                                struct site *site, void *session)
 1058 {
 1059 #if NE_VERSION_MINOR > 25
 1060     struct ne_md5_ctx *md5;
 1061 #define MD5_PTR md5
 1062 #else
 1063     struct ne_md5_ctx md5;
 1064 #define MD5_PTR &md5
 1065 #endif
 1066     char *full_remote = ne_concat(site->remote_root, file->filename, NULL);
 1067     int ret = 0;
 1068 
 1069 #if NE_VERSION_MINOR > 25
 1070     md5 = ne_md5_create_ctx();
 1071 #else
 1072     ne_md5_init_ctx(&md5);
 1073 #endif
 1074 
 1075     fe_checksumming(file->filename);
 1076     if (CALL(file_read)(session, full_remote, 
 1077                         site_fetch_csum_read, MD5_PTR) != SITE_OK) {
 1078         ret = 1;
 1079         fe_checksummed(full_remote, false, DRIVER_ERR);
 1080     } else {
 1081         ne_md5_finish_ctx(MD5_PTR, file->checksum);
 1082         fe_checksummed(full_remote, true, NULL);
 1083     }
 1084     free(full_remote);
 1085 
 1086 #if NE_VERSION_MINOR > 25
 1087     ne_md5_destroy_ctx(md5);
 1088 #endif
 1089 
 1090     return ret;
 1091 }
 1092     
 1093 /* Updates the remote file list... site_fetch_callback is called for
 1094  * every remote file found.
 1095  */
 1096 int site_fetch(struct site *site)
 1097 {
 1098     int ret, need_modtimes;
 1099     void *session;
 1100     const char *dirstack[DIRSTACKSIZE];
 1101     size_t dirtop;
 1102     struct proto_file *files = NULL;
 1103 
 1104     ret = proto_init(site, &session);
 1105     if (ret != SITE_OK) {
 1106     proto_finish(site, session);
 1107     return ret;
 1108     }
 1109 
 1110     if (CALL(fetch_list) == NULL) {
 1111     proto_finish(site, session);
 1112     return SITE_UNSUPPORTED;
 1113     }
 1114 
 1115     /* The remote modtimes are needed if timesize is used or in safe
 1116      * mode: */
 1117     need_modtimes = site->safemode || site->state_method == state_timesize;
 1118 
 1119     dirtop = 1;
 1120     dirstack[0] = "";
 1121 
 1122     do {
 1123         struct proto_file *newfiles = NULL, *f, *lastf = NULL;
 1124         const char *reldir = dirstack[--dirtop];
 1125         const char *slash = reldir[0] == '\0' ? "" : "/";
 1126         char *curdir;
 1127 
 1128         curdir = ne_concat(site->remote_root, reldir, slash, NULL);
 1129 
 1130         ret = CALL(fetch_list)(session, curdir, need_modtimes, &newfiles);
 1131         if (ret != SITE_OK) break;
 1132 
 1133         for (f = newfiles; f; f = f->next) {
 1134             char *relfn;
 1135 
 1136             relfn = ne_concat(reldir, slash, f->filename, NULL);
 1137             ne_free(f->filename);
 1138             f->filename = relfn;
 1139 
 1140             if (!file_isexcluded(relfn, site)) {
 1141                 if (f->type == proto_dir && dirtop < DIRSTACKSIZE) {
 1142                     dirstack[dirtop++] = relfn;
 1143                 } else if (f->type == proto_file 
 1144                            && site->state_method == state_checksum) {
 1145                     fetch_checksum_file(f, site, session);
 1146                 }
 1147             }
 1148 
 1149             lastf = f;
 1150         }
 1151 
 1152         if (lastf) {
 1153             lastf->next = files;
 1154             files = newfiles;
 1155         }
 1156 
 1157         ne_free(curdir);
 1158     } while (dirtop > 0);
 1159     
 1160     if (ret == SITE_OK) {
 1161         struct proto_file *f, *nextf;
 1162 
 1163         /* Remove existing stored state for the site. */
 1164         site_destroy_stored(site);
 1165 
 1166         /* And replace it with the fetched state. */
 1167         for (f = files; f; f = nextf) {
 1168             if (!file_isexcluded(f->filename, site)) {
 1169                 struct site_file *sf = fetch_add_file(site, f);
 1170                 fe_fetch_found(sf);
 1171             }
 1172             nextf = f->next;
 1173             ne_free(f);
 1174         }
 1175     } else {
 1176         ret = SITE_FAILED;
 1177     }
 1178 
 1179     proto_finish(site, session);
 1180     
 1181     return ret;
 1182 }
 1183 
 1184 /* Compares files list with files.
 1185  * Returns SITE_OK on match, SITE_ERRORS on no match.
 1186  */
 1187 /* Ahhhh, this is crap too.
 1188  * If we had a generic file_set this would be easy and clean and spot
 1189  * moved files too. We need a generic file_set.
 1190  */
 1191 static int site_verify_compare(struct site *site, 
 1192                    const struct proto_file *files,
 1193                    int *numremoved)
 1194 {
 1195     struct site_file *file;
 1196     const struct proto_file *lfile;
 1197     int numremote = 0;
 1198 
 1199     /* Clear live state */
 1200     for_each_file(file, site) {
 1201     if (file->stored.exists) {
 1202         numremote++;
 1203     }
 1204     }
 1205 
 1206     for (lfile = files; lfile != NULL; lfile = lfile->next) {
 1207     enum file_diff diff = file_new;
 1208 
 1209     numremote--;
 1210     for_each_file(file, site) {
 1211         if (file->stored.exists &&
 1212         (strcmp(file->stored.filename, lfile->filename) == 0)) {
 1213         /* Do a mini file_compare job */
 1214         diff = file_unchanged;
 1215         if (site->state_method == state_checksum) {
 1216             if (memcmp(file->stored.checksum, lfile->checksum, 16))
 1217             diff = file_changed;
 1218         } else {
 1219             if ((file->stored.size != lfile->size) ||
 1220             (site->safemode && 
 1221              (file->server.time != lfile->modtime))) {
 1222             diff = file_changed;
 1223             }
 1224         }
 1225         break;
 1226         }
 1227     }
 1228     
 1229     /* If new files were added, adjust the count */
 1230     if (diff == file_new)
 1231         numremote++;
 1232 
 1233     fe_verified(lfile->filename, diff); 
 1234     }
 1235 
 1236     *numremoved = numremote;
 1237 
 1238     if (numremote != 0) {
 1239     return SITE_ERRORS;
 1240     } else {
 1241     return SITE_OK;
 1242     }      
 1243 
 1244 }
 1245 
 1246 /* Compares what's on the server with what we THINK is on the server.
 1247  * Returns SITE_OK if match, SITE_ERRORS if doesn't match.
 1248  */
 1249 int site_verify(struct site *site, int *numremoved)
 1250 {
 1251     struct proto_file *files = NULL;
 1252     void *session;
 1253     int ret;
 1254 
 1255     ret = proto_init(site, &session);
 1256     if (ret != SITE_OK)
 1257     return ret;
 1258 
 1259     if (CALL(fetch_list) == NULL) {
 1260     return SITE_UNSUPPORTED;
 1261     }
 1262 
 1263     ret = CALL(fetch_list)(session, site->remote_root, 1, &files);
 1264 
 1265 #if 0
 1266     if (site->state_method == state_checksum) {
 1267     site_fetch_checksum(files, site, session);
 1268     }
 1269 #endif
 1270 
 1271     proto_finish(site, session);
 1272     
 1273     if (ret == SITE_OK) {
 1274     /* Return whether they matched or not */
 1275     return site_verify_compare(site, files, numremoved);
 1276     } else {
 1277     return SITE_FAILED;
 1278     }
 1279 
 1280 }
 1281 
 1282 /* Destroys the stored state of files in the files list for the given
 1283  * site. Removes any files which do not exist locally from the list. */
 1284 void site_destroy_stored(struct site *site)
 1285 {
 1286     struct site_file *current, *next;
 1287     current = site->files;
 1288     while (current != NULL) {
 1289     next = current->next;
 1290     if (!current->local.exists) {
 1291         /* It doesn't exist locally... nuke it */
 1292         file_delete(site, current);
 1293     } else {
 1294         /* Just nuke the stored state...
 1295          * TODO-ngm: verify this. */
 1296         file_state_destroy(&current->stored);
 1297         /* Could just do .exists = false */
 1298         memset(&current->stored, 0, sizeof(struct file_state));
 1299         /* And set the diff */
 1300         file_set_diff(current, site);
 1301     }
 1302     current = next;
 1303     }
 1304 }
 1305 
 1306 /* Called to delete all the files associated with the site */
 1307 void site_destroy(struct site *site)
 1308 {
 1309     struct site_file *current, *next;
 1310 
 1311     current = site->files;
 1312     while (current != NULL) {
 1313     next = current->next;
 1314     file_delete(site, current);
 1315     current = next;
 1316     }
 1317 
 1318 }
 1319 
 1320 
 1321 /* Produces a section of the flat listing output, of all the items
 1322  * with the given diff type in the given site, using the given section
 1323  * name. */
 1324 static void site_flatlist_items(FILE *f, struct site *site,
 1325                                 enum file_diff diff, const char *name)
 1326 {
 1327     struct site_file *current;
 1328     fprintf(f, "sectstart|%s", name);
 1329     putc('\n', f);
 1330     for_each_file(current, site) {
 1331     if (current->diff == diff) {
 1332         fprintf(f, "item|%s%s", file_name(current),
 1333             (current->type==file_dir)?"/":"");
 1334         if (current->diff == file_moved) {
 1335         fprintf(f, "|%s", current->stored.filename);
 1336         }
 1337             if (current->ignore)
 1338                 fputs("|ignored", f);
 1339             putc('\n', f);
 1340     }
 1341     }
 1342     fprintf(f, "sectend|%s\n", name);
 1343 }
 1344 
 1345 /* Produce the flat listing output for the given site */
 1346 void site_flatlist(FILE *f, struct site *site)
 1347 {
 1348     fprintf(f, "sitestart|%s", site->name);
 1349     if (site->url)  fprintf(f, "|%s", site->url);
 1350     putc('\n', f);
 1351     if (site->numnew > 0)
 1352     site_flatlist_items(f, site, file_new, "added");
 1353     if (site->numchanged > 0)
 1354     site_flatlist_items(f, site, file_changed, "changed");
 1355     if (site->numdeleted > 0)
 1356     site_flatlist_items(f, site, file_deleted, "deleted");
 1357     if (site->nummoved > 0)
 1358     site_flatlist_items(f, site, file_moved, "moved");
 1359     fprintf(f, "siteend|%s\n", site->remote_is_different?"changed":"unchanged");
 1360 }
 1361 
 1362 void site_sock_progress_cb(void *userdata, ne_off_t progress, ne_off_t total)
 1363 {
 1364     fe_transfer_progress(progress, total);
 1365 }
 1366 
 1367 void fe_initialize(void)
 1368 {
 1369     ne_sock_init();
 1370 }