"Fossies" - the Fresh Open Source Software Archive

Member "Pansophica-src-1.3/BAF_modules/analysis/src/analyze.c" (1 Feb 2008, 61449 Bytes) of package /linux/www/old/Pansophica-src-1.3-1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "analyze.c" see the Fossies "Dox" file reference documentation.

    1 /******************************************************************************
    2 
    3  Pansophica, An intelligent, virtual-reality, web search agent
    4  Copyright (C) MMVIII, NeuralVR Technologies Ltd.
    5 
    6  This program is free software: you can redistribute it and/or modify
    7  it under the terms of the GNU General Public License as published by
    8  the Free Software Foundation, either version 3 of the License, or
    9  (at your option) any later version.
   10 
   11  This program is distributed in the hope that it will be useful,
   12  but WITHOUT ANY WARRANTY; without even the implied warranty of
   13  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   14  GNU General Public License for more details.
   15 
   16  You should have received a copy of the GNU General Public License
   17  along with this program.  If not, see http://www.gnu.org/licenses/.
   18 
   19  MODULE: analyze.c
   20  $Id: analyze.c,v 1.15 2008/02/01 18:23:07 dean Exp $
   21 
   22  HISTORY
   23  $Log: analyze.c,v $
   24  Revision 1.15  2008/02/01 18:23:07  dean
   25  first GPL, mac ready
   26 
   27  Revision 1.14  2003/11/14 04:29:01  dean
   28  ERROR and CLEANUP tags with BAF_ prefix
   29 
   30  Revision 1.13  2003/11/13 20:18:05  dean
   31  removal of generic macros that collide on Windows, BAF_ prefix
   32 
   33  Revision 1.12  2003/11/12 19:07:10  dean
   34  boolean to BAF_BOOLEAN, true to BAF_TRUE, false to BAF_FALSE
   35 
   36  Revision 1.11  2003/10/22 03:55:14  dean
   37  updated copyrighting licensing
   38 
   39  Revision 1.10  1999/10/16 00:28:44  dean
   40  compiler warning and setting user_data field to NULL in session status set
   41 
   42  Revision 1.9  1999/10/14 02:55:26  dean
   43  searching and aggregation now performed outside module
   44 
   45  Revision 1.8  1999/03/31 21:58:52  dean
   46  returning 'no documents found' error
   47 
   48  Revision 1.7  1999/03/17 23:51:14  dean
   49  better handling of info mode string
   50 
   51  Revision 1.6  1999/03/17 22:21:47  dean
   52  added searching
   53 
   54  Revision 1.5  1999/02/18 01:05:03  dean
   55  save the recall file before returning the results
   56 
   57  Revision 1.4  1999/02/15 21:06:31  dean
   58  remove the recall file on cancelation or failure
   59 
   60  Revision 1.3  1999/01/27 23:34:26  dean
   61  Adding dna/ genome information to the recall file
   62 
   63  Revision 1.2  1999/01/26 20:43:33  dean
   64  cleanup of util_mem_free
   65 
   66  Revision 1.1.1.1  1999/01/21 20:42:27  dean
   67  initial import of the new BAF_modules
   68 
   69  Revision 1.24  1998/12/04 18:30:19  dean
   70  Added NPL notice
   71 
   72  Revision 1.23  1998/10/09 16:55:10  dean
   73  smaller recall file
   74 
   75  Revision 1.22  1998/09/21 14:49:35  dean
   76  added name of world to status blocks
   77 
   78  Revision 1.21  1998/09/08 16:01:26  dean
   79  error string allocation was too small
   80 
   81  Revision 1.20  1998/08/26 14:35:08  dean
   82  control of recursion level on fetch
   83 
   84  Revision 1.19  1998/08/24 15:23:51  dean
   85  fixed two memory leaks
   86 
   87  Revision 1.18  1998/07/24 16:13:06  dean
   88  stats container removed because of a few hitches with recall.c
   89 
   90  Revision 1.17  1998/07/22 15:55:33  dean
   91  fixed closure of recall file, now calling with address of ref
   92 
   93  Revision 1.16  1998/07/15 14:42:29  dean
   94  returning recall file and world title; fixed up return strings
   95 
   96  Revision 1.15  1998/07/06 14:31:28  dean
   97  new macros, tested
   98 
   99  Revision 1.14  1998/06/22 14:32:38  dean
  100  passing through option to skip fetch if site already fetched
  101 
  102  Revision 1.13  1998/06/16 23:39:21  dean
  103  Removed debug
  104 
  105  SYNOPSIS
  106  Master module for acquisition, comprehension and organization
  107 
  108  HISTORY
  109  Created May 16, 98 by dab
  110 
  111  NOTES
  112  Actions:
  113      "new"    - create a new world, starting with given origin
  114      "add"    - add a new origin to an existing world
  115      "remix"  - remake the world
  116      "update" - update the given origin of the given world
  117  
  118  new
  119  ---
  120  Make a new world with the given origin.
  121 
  122  Takes the following whack of args:
  123  world ...
  124      s - path to recall file
  125      s - title of this world
  126      s - temp path ( for dna, plane, docs, other work files )
  127  genesis ...
  128      s - path to file of no words ( words to ignore of format # word )
  129      s - path to file of yes words ( words to consider ) as above
  130  origin ...
  131      s - title of origin
  132      s - machine description
  133      s - base url
  134      s - username of person performing this op
  135      s - user theme
  136      s - local path to which to fetch
  137  acquisition ...
  138      s - name of acquisition module ( ie. web_acquire )
  139      s - http proxy, if any
  140      s - ftp proxy, if any, likely same as http proxy
  141      s - name of get module
  142      s - get module action
  143      s - file extensions
  144      d - recursion level for fetching
  145      s - exclude domains
  146  info ...
  147      s - ending/ cleaner map file
  148  ft_comprehend ( dna ) ...
  149      d - number of keys per file
  150      f - target q_error ( as proportion of initial q, suggest 0.4 or .5 )
  151      d - max training runs
  152      d - max training time
  153      s - dna params mod
  154      f - density multiplier
  155  ft_organize ...
  156      f - target q_error
  157      d - max training runs
  158      d - max training time
  159      s - docs params mod
  160      f - density multiplier
  161      d - umat averaging ( use averaging to smooth umat )
  162      d - umat median ( use the median calcs to smooth the umat )
  163      d - fingerprint reduction factor ( red fingerprint size by this fac )
  164 
  165  Returns the following measures of success and performance:
  166      s - path to outputted recall file
  167      s - world title as passed in
  168      d - total files
  169      d - total bytes
  170      d - total seconds
  171      d - total files got
  172      d - total files acquired
  173      d - seconds for fetching
  174      d - total files infoed
  175      d - seconds for infoing
  176      d - total concepts comprehended ( lin )
  177      d - total concept contexts ( din )
  178      d - seconds for comprehension
  179      d - seconds on training in comprehension
  180      f - q_error for comprehension
  181      d - xdim of dna
  182      d - ydim of dna
  183      d - total labels on plane
  184      d - total docs organized
  185      d - total number of clusters ( plane dimensions )
  186      d - seconds for organization
  187      d - seconds for training in organization
  188      f - q_error for organizaion
  189      d - xdim of plane
  190      d - ydim of plane
  191 
  192  new_search
  193  ----------
  194  As per 'new' above, with the following input parameters added at the end:
  195  search ...
  196      s - search url
  197      s - search file
  198 
  199  Same outputs as 'new' above.
  200 
  201 
  202  create TOBEDONE
  203  ------
  204  A quickie to create and empty world if we ever need one.  For the demo,
  205  we're interested in seeing things so we'll come back to this later.  Also,
  206  This will likely lend itself to a lot of remixing as folks add one doc
  207  at a time to world ( bookmarking ) ... so later ...
  208 
  209 
  210  add TOBEDONE ... params and returns need some work too ...
  211  --- 
  212  Add an origin to an existing file.  No dna or plane remix.
  213 
  214 
  215  update TOBEDONE ... params and returns need some work too ...
  216  ------
  217  Update the given origin in the given recall file.  New and modified docs
  218  are organized.  No dna or plane remix.
  219 
  220 
  221  remix TOBEDONE ... look at ft_comprehend and ft_organize
  222  -----
  223  Remake the soms and output ... by stage ...
  224 
  225 ******************************************************************************/
  226 
  227 #include <stdio.h>
  228 #include <time.h>
  229 #include <float.h>
  230 #include <math.h>
  231 #include <string.h>
  232 #include "modules_defs.h"
  233 #include "cn_defs.h"
  234 
  235 /* : our actions */
  236 #define AZ_MY_NAME "analyze"
  237 #define AZ_NEW "new"
  238 #define AZ_ADD "add"
  239 #define AZ_REMIX "remix"
  240 #define AZ_UPDATE "update"
  241 #define AZ_NEW_SEARCH "new_search"
  242 
  243 #define AZ_INFO_MODULE "in_all"
  244 #define AZ_COMPREHEND_MODULE "ft_comprehend"
  245 #define AZ_ORGANIZE_MODULE "ft_organize"
  246 
  247 /* : some constants */
  248 #define AZ_ACQUIRE_NAME "acquire"
  249 #define AZ_DNA_NAME "dna"
  250 #define AZ_DOCS_NAME "docs"
  251 #define AZ_PLANE_NAME "plane"
  252 
  253 /* : errors */
  254 typedef enum {
  255   AZ_OK,
  256   AZ_FAIL,
  257   AZ_CANT_ALLOCATE_INFO,
  258   AZ_BAD_PARAMS,
  259   AZ_CANT_SEARCH,
  260   AZ_CANT_PREP,
  261   AZ_CANT_ACQUIRE,
  262   AZ_NO_DOCUMENTS_FOUND,
  263   AZ_CANT_INFO,
  264   AZ_CANT_COMPREHEND,
  265   AZ_CANT_WRITE_GENOME,
  266   AZ_CANT_ORGANIZE,
  267   AZ_CANT_WRITE_RECALL,
  268   AZ_CANT_REPLY,
  269   AZ_CANT_HANDLE_RESPONSE,
  270   AZ_UNSUPPORTED
  271 } AZ_RES;
  272 
  273 /* : error strings */
  274 static char *s_error[] = {
  275   "az no error",
  276   "az general failure",
  277   "az can't allocate info",
  278   "az bad params",
  279   "az can't search",
  280   "az can't create and set up recall file",
  281   "az can't acquire",
  282   "az no documents found",
  283   "az can't info",
  284   "az can't comprehend",
  285   "az can't write genome",
  286   "az can't organize"
  287   "az can't write recall file",
  288   "az can't reply",
  289   "az can't handle response",
  290   "az unsupported"
  291 };
  292 
  293 /* : info struct for new */
  294 typedef struct {
  295 
  296   /* : params */
  297   struct {          /* params */
  298     struct {            /* params.BAF */
  299       ENDO_MSG_REF msg_ref;
  300     } BAF;
  301     struct {            /* params.world */
  302       char *recall_file;
  303       RECALL_FILE recall_file_ref;
  304       char *title;
  305       char *temp_path;
  306     } world;
  307     struct {            /* params.genesis */
  308       char *no_path;
  309       char *yes_path;
  310     } genesis;
  311     struct {            /* params.origin */
  312       char *title;
  313       char *machine;
  314       char *base_url;
  315       char *username;
  316       char *user_theme;
  317       char *fetch_path;
  318     } origin;
  319     struct {            /* params.fetch */
  320       char *module;
  321       char *http_proxy;
  322       char *ftp_proxy;
  323       char *get_module;
  324       char *get_action;
  325       char *file_extensions;
  326       int recursion_level;
  327       char *exclude_domains;
  328     } fetch;
  329     struct {            /* params.info */
  330       char *endings_and_cleaners;
  331     } info;
  332     struct {            /* params.comprehend */
  333       int num_keys_per_file;
  334       float target_q_error;
  335       int max_training_runs;
  336       int max_training_seconds;
  337       char *params_mod;
  338       float m_density;
  339     } comprehend;
  340     struct {            /* params.organize */
  341       float target_q_error;
  342       int max_training_runs;
  343       int max_training_seconds;
  344       char *params_mod;
  345       float m_density;
  346       int umat_average;
  347       int umat_median;
  348       int fp_reduction_factor;
  349     } organize;
  350     struct {            /* params.search */
  351       char *url;
  352       char *file;
  353     } search;
  354   } params;
  355 
  356   struct {          /* work */
  357     struct {            /* work.stars */
  358       time_t t_start;
  359       time_t t_fetch;
  360       time_t t_info;
  361       time_t t_comprehend;
  362       time_t t_organize;
  363     } starts;
  364     struct {            /* work.paths */
  365       char *acquire;
  366       char *dna;
  367       char *plane;
  368       char *docs;
  369     } paths;
  370     struct {            /* work.dna */
  371       char *buff;
  372       int size;
  373     } dna;
  374     struct {            /* work.words */
  375       UTIL_TREE_REF t_no;
  376       UTIL_TREE_REF t_yes;
  377     } words;
  378     struct {            /* work.error */
  379       char *str;
  380     } error;
  381     struct {            /* work.misc */
  382       RECALL_CONTAINER c_cont;
  383       RECALL_CONTAINER o_cont;
  384       RECALL_CONTAINER w_cont;
  385       RECALL_CONTAINER g_cont;
  386       char *s_buff;
  387     } misc;
  388   } work;
  389   
  390   struct {          /* results */
  391     struct {            /* results.totals */
  392       int files;
  393       int bytes;
  394       int seconds;
  395     } totals;
  396     struct {            /* results.fetch */
  397       int files_fetched;
  398       int files_acquired;
  399       int seconds;
  400     } fetch;
  401     struct {            /* results.info */
  402       int files;
  403       int seconds;
  404     } info;
  405     struct {            /* results.comprehend */
  406       int concepts;
  407       int contexts;
  408       int seconds;
  409       int training_seconds;
  410       float q_error;
  411       int xdim;
  412       int ydim;
  413     } comprehend;
  414     struct {            /* results.organize */
  415       int labels;
  416       int docs;
  417       int clusters;
  418       int seconds;
  419       int training_seconds;
  420       float q_error;
  421       int xdim;
  422       int ydim;
  423     } organize;
  424   } results;
  425 
  426 } AZ_NEW_INFO, *AZ_NEW_INFO_PTR;
  427 
  428 #define S_BUFSIZ 64
  429 
  430 
  431 /******************************************************************************
  432 
  433  FUNCTION: az_new_new_info
  434 
  435  SYNOPSIS
  436  Allocates a new info structure
  437 
  438  HISTORY
  439  Created May 17, 98 by dab
  440 
  441 ******************************************************************************/
  442 
  443 static AZ_RES
  444 az_new_new_info(
  445 AZ_NEW_INFO_PTR *pp_info
  446 )
  447 {
  448   AZ_NEW_INFO_PTR info = NULL;
  449 
  450   BAF_IN(PRIVATE);
  451 
  452   TAKE( *pp_info ,(AZ_NEW_INFO_PTR), sizeof(AZ_NEW_INFO) ) {
  453     BAF_THROW_MSG("can't allocate info");
  454   }
  455   info = *pp_info;
  456 
  457   TAKE( info->work.misc.s_buff ,(char *), S_BUFSIZ ) {
  458     BAF_THROW_MSG("can't allocate s_buff");
  459   }
  460 
  461   BAF_OUT(PRIVATE);
  462 }
  463 
  464 
  465 /******************************************************************************
  466 
  467  FUNCTION: az_dispose_count_tree
  468 
  469  SYNOPSIS
  470  Disposes a count tree, like the yes and no trees here
  471 
  472  HISTORY
  473  IncrDev Jul  5, 98 by dab: sent outside of session
  474  Created May 17, 98 by dab
  475 
  476  NOTES
  477 
  478 ******************************************************************************/
  479 
  480 static AZ_RES
  481 az_dispose_count_tree(
  482 ENDO_MSG_REF msg_ref,
  483 UTIL_TREE_REF *t_count
  484 )
  485 {
  486   ENDO_MSG_REF response = NULL;
  487   BAF_BOOLEAN is_success = BAF_FALSE;
  488 
  489   BAF_IN(PRIVATE);
  490 
  491   if ( t_count == NULL ) {
  492     BAF_OUT(PRIVATE);
  493   }
  494   if ( *t_count == NULL ) {
  495     BAF_OUT(PRIVATE);
  496   }
  497 
  498   /*  BAF_CATCH( endo_msg_send_simple( msg_ref, */
  499   /* : send this outside the session so that it gets done even if our session
  500      : is canceled */
  501   BAF_CATCH( endo_msg_send_simple( NULL,
  502                    AZ_MY_NAME,
  503                    COUNT_DISPOSE_MODULE,
  504                    COUNT_DISPOSE,
  505                    ENDO_RESPONSE_WAIT,
  506                    &response,
  507                    "p",
  508                    *t_count ) ) {
  509     BAF_THROW_MSG("can't send");
  510   }
  511 
  512   BAF_CATCH( endo_msg_response_is_success( response, &is_success ) ) {
  513     BAF_THROW_MSG("can't determine success or fail");
  514   }
  515 
  516   if ( is_success == BAF_FALSE ) {
  517     BAF_THROW();
  518   }
  519 
  520   /* : zero to guard multiples */
  521   *t_count = NULL;
  522 
  523   BAF_OUT(PRIVATE);
  524 }
  525 
  526 
  527 /******************************************************************************
  528 
  529  FUNCTION: az_new_dispose_info
  530 
  531  SYNOPSIS
  532  Disposes of the info struct
  533 
  534  HISTORY
  535  Created May 17, 1998 by dab
  536  BugFix  Aug 24, 1998 by dab: giving some leaks
  537 
  538 ******************************************************************************/
  539 
  540 static AZ_RES
  541 az_new_dispose_info(
  542 AZ_NEW_INFO_PTR *pp_info
  543 )
  544 {
  545   AZ_NEW_INFO_PTR info = NULL;
  546 
  547   BAF_IN(PRIVATE);
  548 
  549   if ( pp_info == NULL ) {
  550     BAF_OUT(PRIVATE);
  551   }
  552   if ( *pp_info == NULL ) {
  553     BAF_OUT(PRIVATE);
  554   }
  555 
  556   info = *pp_info;
  557 
  558   /* : params */
  559   /* : params.world */
  560   GIVE(info->params.world.recall_file);
  561   if ( info->params.world.recall_file_ref != NULL ) {
  562 
  563     /* : save before replying ...
  564     BAF_CATCH( recall_file_save( info->params.world.recall_file_ref ) ) {
  565       MSG(PRIVATE,"can't save recall file" );
  566     }
  567     */
  568 
  569     BAF_CATCH( recall_file_close( &(info->params.world.recall_file_ref) ) ) {
  570       MSG(PRIVATE,"can't close recall file" );
  571     }
  572 
  573   }
  574 
  575   GIVE(info->params.world.title);
  576   GIVE(info->params.world.temp_path);
  577 
  578   /* : params.genesis */
  579   GIVE(info->params.genesis.no_path);
  580   GIVE(info->params.genesis.yes_path);
  581 
  582   /* : params.origin */
  583   GIVE(info->params.origin.title);
  584   GIVE(info->params.origin.machine);
  585   GIVE(info->params.origin.base_url);
  586   GIVE(info->params.origin.username);
  587   GIVE(info->params.origin.user_theme);
  588   GIVE(info->params.origin.fetch_path);
  589 
  590   /* : params.fetch */
  591   GIVE(info->params.fetch.module);
  592   GIVE(info->params.fetch.http_proxy);
  593   GIVE(info->params.fetch.ftp_proxy);
  594   GIVE(info->params.fetch.get_module);
  595   GIVE(info->params.fetch.get_action);
  596   GIVE(info->params.fetch.file_extensions);
  597   GIVE(info->params.fetch.exclude_domains);
  598 
  599   /* : params.info */
  600   GIVE(info->params.info.endings_and_cleaners);
  601 
  602   /* : params.comprecomprehend */
  603   GIVE(info->params.comprehend.params_mod);
  604 
  605   /* : params.organize */
  606   GIVE(info->params.organize.params_mod);
  607 
  608   /* : params.search */
  609   GIVE(info->params.search.url);
  610   GIVE(info->params.search.file);
  611 
  612   /* : work */
  613   /* : work.paths */
  614   GIVE(info->work.paths.acquire);
  615   GIVE(info->work.paths.dna);
  616   GIVE(info->work.paths.plane);
  617   GIVE(info->work.paths.docs);
  618 
  619   /* : work.dna */
  620   GIVE(info->work.dna.buff);
  621 
  622   /* : work.words.t_no and t_yes */
  623   if ( info->work.words.t_no != NULL ) {
  624     BAF_CATCH( az_dispose_count_tree( info->params.BAF.msg_ref,
  625                   &(info->work.words.t_no) ) ) {
  626       MSG(PRIVATE,"can't dispose no tree" );
  627     }
  628   }
  629   if ( info->work.words.t_yes != NULL ) {
  630     BAF_CATCH( az_dispose_count_tree( info->params.BAF.msg_ref,
  631                   &(info->work.words.t_yes) ) ) {
  632       MSG(PRIVATE,"can't dispose yes tree" );
  633     }
  634   }
  635 
  636   /* : work.error */
  637   GIVE(info->work.error.str);
  638 
  639   /* : work.misc.s_buff */
  640   GIVE(info->work.misc.s_buff);
  641   
  642   GIVE( (*pp_info) );
  643   *pp_info = NULL;
  644 
  645   BAF_OUT(PRIVATE);
  646 }
  647 
  648 
  649 /******************************************************************************
  650 
  651  FUNCTION: az_load_yes_and_no_trees
  652 
  653  SYNOPSIS
  654  Loads the yes and no words into trees
  655 
  656  HISTORY
  657  IncrDev May 19, 98 by dab: now getting and return params
  658  Created May 17, 98 by dab
  659 
  660 ******************************************************************************/
  661 
  662 static AZ_RES
  663 az_load_yes_and_no_trees(
  664 ENDO_MSG_REF msg_ref,
  665 char *no_path,
  666 char *yes_path,
  667 UTIL_TREE_REF *pp_t_no,
  668 UTIL_TREE_REF *pp_t_yes
  669 )
  670 {
  671   ENDO_MSG_REF response = NULL;
  672 
  673   BAF_IN(PRIVATE);
  674 
  675   /* : call the wordlist module to load the trees */
  676   BAF_CATCH( endo_msg_send_simple( msg_ref,
  677                    AZ_MY_NAME,
  678                    COUNT_WORDLIST_MODULE,
  679                    COUNT_WORDLIST_READ,
  680                    ENDO_RESPONSE_WAIT,
  681                    &response,
  682                    "s",
  683                    no_path ) ) {
  684     BAF_THROW_MSG("can't send");
  685   }
  686 
  687   BAF_CATCH( endo_msg_response_get_args( response, "p", pp_t_no ) ) {
  688     BAF_THROW_MSG("can't get response args");
  689   }
  690     
  691   BAF_CATCH( endo_msg_send_simple( msg_ref,
  692                    AZ_MY_NAME,
  693                    COUNT_WORDLIST_MODULE,
  694                    COUNT_WORDLIST_READ,
  695                    ENDO_RESPONSE_WAIT,
  696                    &response,
  697                    "s",
  698                    yes_path ) ) {
  699     BAF_THROW_MSG("can't send to load yes words");
  700   }
  701 
  702   BAF_CATCH( endo_msg_response_get_args( response, "p", pp_t_yes ) ) {
  703     BAF_THROW_MSG("can't get response args for yes words");
  704   }
  705 
  706   BAF_OUT(PRIVATE);
  707 }
  708 
  709 
  710 /******************************************************************************
  711 
  712  FUNCTION: az_append_file_to_dir
  713 
  714  SYNOPSIS
  715  Allocates and appends a file to a dir to create a full path
  716 
  717  HISTORY
  718  Created May 17, 98 by dab
  719 
  720 ******************************************************************************/
  721 
  722 static AZ_RES
  723 az_append_file_to_dir(
  724 char *dir,
  725 char *file,
  726 char **pp_path
  727 )
  728 {
  729   BAF_IN(PRIVATE);
  730 
  731   TAKE( *pp_path ,(char *), strlen(dir) + strlen(file) + 1 ) {
  732     BAF_THROW();
  733   }
  734   sprintf( *pp_path, "%s%s", dir, file );
  735 
  736   BAF_OUT(PRIVATE);
  737 }
  738 
  739 
  740 /******************************************************************************
  741 
  742  FUNCTION: az_get_intermediate_paths
  743 
  744  SYNOPSIS
  745  Sets the intermediate paths
  746 
  747  HISTORY
  748  IncrDev May 18, 98 by dab: allowing null params
  749  Created May 17, 98 by dab
  750 
  751  NOTE
  752  All in one right now
  753 
  754 ******************************************************************************/
  755 
  756 static AZ_RES
  757 az_get_intermediate_paths(
  758 char *temp_path,
  759 char **pp_acquire,
  760 char **pp_dna,
  761 char **pp_plane,
  762 char **pp_docs
  763 )
  764 {
  765   BAF_IN(PRIVATE);
  766 
  767   /* : acquire */
  768   if ( pp_dna != NULL ) {
  769     BAF_CATCH( az_append_file_to_dir( temp_path, AZ_ACQUIRE_NAME, pp_acquire ) ) {
  770       BAF_ERROR_MSG("can't set acquire path" );
  771     }
  772   }
  773 
  774   /* : dna */
  775   if ( pp_dna != NULL ) {
  776     BAF_CATCH( az_append_file_to_dir( temp_path, AZ_DNA_NAME, pp_dna ) ) {
  777       BAF_ERROR_MSG("can't set dna path" );
  778     }
  779   }
  780 
  781   /* : plane */
  782   if ( pp_plane != NULL ) {
  783     BAF_CATCH( az_append_file_to_dir( temp_path, AZ_PLANE_NAME, pp_plane ) ) {
  784       BAF_ERROR_MSG("cant' set plane path" );
  785     }
  786   }
  787 
  788   /* : docs */
  789   if ( pp_docs != NULL ) {
  790     BAF_CATCH( az_append_file_to_dir( temp_path, AZ_DOCS_NAME, pp_docs ) ) {
  791       BAF_ERROR_MSG("can't set docs path" );
  792     }
  793   }
  794 
  795   BAF_OUT(PRIVATE);
  796 
  797 
  798  BAF_ERROR:
  799 
  800   /* : double-level here as we can pass in NULL's */
  801   if ( pp_acquire != NULL ) {
  802     GIVE( (*pp_acquire) );
  803   }
  804   if ( pp_dna != NULL ) {
  805     GIVE( (*pp_dna) );
  806   }
  807   if ( pp_plane != NULL ) {
  808     GIVE( (*pp_plane) );
  809   }
  810   if ( pp_docs != NULL ) {
  811     GIVE( (*pp_docs) );
  812   }
  813 
  814   BAF_THROW();
  815 }
  816 
  817 
  818 /******************************************************************************
  819 
  820  FUNCTION: az_new_prep
  821 
  822  SYNOPSIS
  823  Prep for new
  824 
  825  HISTORY
  826  Created May 17, 98 by dab
  827 
  828 ******************************************************************************/
  829 
  830 static AZ_RES
  831 az_new_prep(
  832 AZ_NEW_INFO_PTR info
  833 )
  834 {
  835   BAF_IN(PRIVATE);
  836 
  837   /* : create the world file */
  838   BAF_CATCH( recall_file_new( info->params.world.recall_file, 
  839               &(info->params.world.recall_file_ref) ) ) {
  840     BAF_THROW_MSG("can't create recall file" );
  841   }
  842 
  843   /* : create the 'contents' container, a container for origins */
  844   BAF_CATCH( recall_container_new( info->params.world.recall_file_ref, 
  845                   NULL, "contents", &(info->work.misc.c_cont) ) ) {
  846     BAF_THROW_MSG("can't create contents container" );
  847   }
  848 
  849   /* : create the origin inside the contents container */
  850   BAF_CATCH( recall_container_new( info->params.world.recall_file_ref, 
  851      info->work.misc.c_cont, "origin", &(info->work.misc.o_cont) ) ) {
  852     BAF_THROW_MSG("can't create origin container" );
  853   }
  854 
  855   /* : create the world */
  856   BAF_CATCH( recall_container_new( info->params.world.recall_file_ref, 
  857                    NULL, "world", &(info->work.misc.w_cont) ) ) {
  858     BAF_THROW_MSG("can't create world container" );
  859   }
  860 
  861   /* : create the genesis */
  862   BAF_CATCH( recall_container_new( info->params.world.recall_file_ref, 
  863                    NULL, "genesis", &(info->work.misc.g_cont) ) ) {
  864     BAF_THROW_MSG("can't create genesis container" );
  865   }
  866 
  867   /* : load the yes and no words into a tree */
  868   BAF_CATCH( az_load_yes_and_no_trees( info->params.BAF.msg_ref, 
  869                    info->params.genesis.no_path,
  870                    info->params.genesis.yes_path,
  871                    &(info->work.words.t_no),
  872                    &(info->work.words.t_yes) ) ) {
  873     BAF_THROW_MSG("can't load yes and no trees" );
  874   }
  875 
  876   /* : the intermediate paths */
  877   BAF_CATCH( az_get_intermediate_paths( info->params.world.temp_path,
  878                     &(info->work.paths.acquire),
  879                     &(info->work.paths.dna),
  880                     &(info->work.paths.plane),
  881                     &(info->work.paths.docs) ) ) {
  882     BAF_THROW_MSG("can't set intermediate paths" );
  883   }
  884 
  885   BAF_OUT(PRIVATE);
  886 }
  887 
  888 
  889 /******************************************************************************
  890 
  891  FUNCTION: az_new_get_response_err
  892 
  893  SYNOPSIS
  894  Fetches an error string if there is one
  895 
  896  HISTORY
  897  Created May 19, 98 by dab
  898 
  899 ******************************************************************************/
  900 
  901 static AZ_RES
  902 az_new_get_response_err_str(
  903 char **pp_str,
  904 ENDO_MSG_REF response
  905 )
  906 {
  907   BAF_IN(PRIVATE);
  908 
  909   BAF_CATCH( endo_msg_response_get_error_string( response, pp_str ) ) {
  910     BAF_THROW();
  911   }
  912 
  913   BAF_OUT(PRIVATE);
  914 }
  915 
  916 
  917 /******************************************************************************
  918 
  919  FUNCTION: az_new_fetch
  920 
  921  SYNOPSIS
  922  Fetches content with the given fetcher module
  923 
  924  HISTORY
  925  Created May 18, 98 by dab
  926 
  927 ******************************************************************************/
  928 
  929 static AZ_RES
  930 az_new_fetch(
  931 AZ_NEW_INFO_PTR info
  932 )
  933 {
  934   ENDO_MSG_REF response = NULL;
  935 
  936   BAF_IN(PRIVATE);
  937 
  938   /* : mark the start time */
  939   info->work.starts.t_fetch = time(NULL);
  940 
  941   BAF_CATCH( endo_msg_send_simple( info->params.BAF.msg_ref,
  942                 AZ_MY_NAME,
  943                 info->params.fetch.module,
  944                 AZ_NEW,
  945                 ENDO_RESPONSE_WAIT,
  946                 &response,
  947                 "sssdssssssds",
  948                 info->params.world.recall_file,
  949                 info->params.fetch.get_module,
  950                 info->params.fetch.get_action,
  951                 1,
  952                 info->params.origin.base_url,
  953                 info->params.fetch.file_extensions,
  954                 info->params.origin.fetch_path,
  955                 info->params.fetch.http_proxy,
  956                 info->params.fetch.ftp_proxy,
  957                 info->work.paths.acquire,
  958                 info->params.fetch.recursion_level,
  959                 info->params.fetch.exclude_domains) ) {
  960     BAF_THROW_MSG("can't send");
  961    }
  962 
  963    BAF_CATCH( endo_msg_response_get_args( response, "ddd",
  964                       &(info->results.fetch.files_fetched),
  965                       &(info->results.fetch.files_acquired),
  966                       &(info->results.totals.bytes) ) ) {
  967      /* : fetch the error string and number into our struct */
  968      BAF_CATCH( az_new_get_response_err_str( &(info->work.error.str), response )) {
  969        MSG(PRIVATE,"can't capture response error string");
  970      }
  971      BAF_THROW_MSG("can't get response args");
  972    }
  973 
  974    /* : our more global indicators */
  975    info->results.totals.files = info->results.fetch.files_acquired;
  976    info->results.fetch.seconds = (int)difftime( time(NULL), 
  977                         info->work.starts.t_fetch );
  978 
  979    BAF_OUT(PRIVATE);
  980 }
  981 
  982 
  983 /******************************************************************************
  984 
  985  FUNCTION: az_new_info
  986 
  987  SYNOPSIS
  988  Performs infoing as part of new
  989 
  990  HISTORY
  991  Created May 18, 98 by dab
  992 
  993 ******************************************************************************/
  994 
  995 static AZ_RES
  996 az_new_info(
  997 AZ_NEW_INFO_PTR info,
  998 char *mode
  999 )
 1000 {
 1001   ENDO_MSG_REF response = NULL;
 1002 
 1003   BAF_IN(PRIVATE);
 1004 
 1005   /* : mark the start time */
 1006   info->work.starts.t_info = time(NULL);
 1007 
 1008   BAF_CATCH( endo_msg_send_simple( info->params.BAF.msg_ref,
 1009                    AZ_MY_NAME,
 1010                    AZ_INFO_MODULE,
 1011                    AZ_NEW,
 1012                    ENDO_RESPONSE_WAIT,
 1013                    &response,
 1014                    "ssss",
 1015                    info->params.world.recall_file,
 1016                    info->params.info.endings_and_cleaners,
 1017                    info->params.origin.username,
 1018                    mode ) ) {
 1019     BAF_THROW_MSG("can't send");
 1020   }
 1021 
 1022   BAF_CATCH( endo_msg_response_get_args( response, "d",
 1023                      &(info->results.info.files ) ) ) {
 1024     /* : fetch the error string and number into our struct */
 1025     BAF_CATCH( az_new_get_response_err_str( &(info->work.error.str), response ) ) {
 1026       MSG(PRIVATE,"can't get response string");
 1027     }
 1028     BAF_THROW_MSG("can't get response args");
 1029   }
 1030   
 1031   /* : mark the duration */
 1032   info->results.info.seconds = (int)difftime( time(NULL), 
 1033                           info->work.starts.t_info );
 1034 
 1035   BAF_OUT(PRIVATE);
 1036 }
 1037 
 1038 
 1039 /******************************************************************************
 1040 
 1041  FUNCTION: az_new_comprehend
 1042 
 1043  SYNOPSIS
 1044  Performs the comprehend stage for a new
 1045 
 1046  HISTORY
 1047  Created May 18, 98 by dab
 1048 
 1049 ******************************************************************************/
 1050 
 1051 static AZ_RES
 1052 az_new_comprehend(
 1053 AZ_NEW_INFO_PTR info
 1054 )
 1055 {
 1056   ENDO_MSG_REF response = NULL;
 1057 
 1058   BAF_IN(PRIVATE);
 1059 
 1060   /* : mark the start time */
 1061   info->work.starts.t_comprehend = time(NULL);
 1062 
 1063   BAF_CATCH( endo_msg_send_simple(info->params.BAF.msg_ref,
 1064                   AZ_MY_NAME,
 1065                   AZ_COMPREHEND_MODULE,
 1066                   AZ_NEW,
 1067                   ENDO_RESPONSE_WAIT,
 1068                   &response,
 1069                   "sppdfddsfs",
 1070                   info->params.world.recall_file,
 1071                   info->work.words.t_no,
 1072                   info->work.words.t_yes,
 1073                   info->params.comprehend.num_keys_per_file,
 1074                   info->params.comprehend.target_q_error,
 1075                   info->params.comprehend.max_training_runs,
 1076                   info->params.comprehend.max_training_seconds,
 1077                   info->params.comprehend.params_mod,
 1078                   info->params.comprehend.m_density,
 1079                   info->work.paths.dna ) ) {
 1080     BAF_THROW_MSG("can't send");
 1081   }
 1082 
 1083   BAF_CATCH( endo_msg_response_get_args
 1084      ( response,
 1085        "pdddddfd",
 1086        &(info->work.dna.buff),
 1087        &(info->work.dna.size),
 1088        &(info->results.comprehend.concepts),
 1089        &(info->results.comprehend.contexts),
 1090        &(info->results.comprehend.xdim),
 1091        &(info->results.comprehend.ydim),
 1092        &(info->results.comprehend.q_error),
 1093        &(info->results.comprehend.training_seconds) ) ) {
 1094 
 1095     /* : fetch the error string and number into our struct */
 1096     BAF_CATCH( az_new_get_response_err_str( &(info->work.error.str), response )) {
 1097       MSG(PRIVATE,"can't get response string");
 1098     }
 1099     BAF_THROW_MSG("can't get response args");
 1100 
 1101   }
 1102   
 1103   /* : mark the duration */
 1104   info->results.comprehend.seconds = (int)difftime( time(NULL), 
 1105                           info->work.starts.t_comprehend );
 1106 
 1107   BAF_OUT(PRIVATE);
 1108 }
 1109 
 1110 
 1111 /******************************************************************************
 1112 
 1113  FUNCTION: az_new_organize
 1114 
 1115  SYNOPSIS
 1116  Performs the analysis stage for new
 1117 
 1118  HISTORY
 1119  Created May 19, 98 by dab
 1120 
 1121 ******************************************************************************/
 1122 
 1123 static AZ_RES
 1124 az_new_organize(
 1125 AZ_NEW_INFO_PTR info
 1126 )
 1127 {
 1128   ENDO_MSG_REF response = NULL;
 1129 
 1130   BAF_IN(PRIVATE);
 1131 
 1132   /* : mark the start time */
 1133   info->work.starts.t_organize = time(NULL);
 1134 
 1135   BAF_CATCH( endo_msg_send_simple(info->params.BAF.msg_ref,
 1136                   AZ_MY_NAME,
 1137                   AZ_ORGANIZE_MODULE,
 1138                   AZ_NEW,
 1139                   ENDO_RESPONSE_WAIT,
 1140                   &response,
 1141                   "sfddsfdddpdppss",
 1142                   info->params.world.recall_file,
 1143                   info->params.organize.target_q_error,
 1144                   info->params.organize.max_training_runs,
 1145                   info->params.organize.max_training_seconds,
 1146                   info->params.organize.params_mod,
 1147                   info->params.organize.m_density,
 1148                   info->params.organize.umat_average,
 1149                   info->params.organize.umat_median,
 1150                   info->params.organize.fp_reduction_factor,
 1151                   info->work.dna.buff,
 1152                   info->work.dna.size,
 1153                   info->work.words.t_no,
 1154                   info->work.words.t_yes,
 1155                   info->work.paths.plane,
 1156                   info->work.paths.docs ) ) {
 1157     BAF_THROW_MSG("can't send");
 1158   }
 1159 
 1160   BAF_CATCH( endo_msg_response_get_args
 1161      ( response,
 1162        "ddddfd",
 1163        &(info->results.organize.labels),
 1164        &(info->results.organize.docs),
 1165        &(info->results.organize.xdim),
 1166        &(info->results.organize.ydim),
 1167        &(info->results.organize.q_error),
 1168        &(info->results.organize.training_seconds) ) ) {
 1169     /* : fetch the error string and number into our struct */
 1170     BAF_CATCH(az_new_get_response_err_str( &(info->work.error.str), response )) {
 1171       MSG(PRIVATE,"can't get response string");
 1172     }
 1173     BAF_THROW_MSG("can't get response args");
 1174   }
 1175 
 1176   /* : calc this one quickly */
 1177   info->results.organize.clusters = info->results.organize.xdim *
 1178                                     info->results.organize.ydim;
 1179   
 1180   /* : mark the duration */
 1181   info->results.organize.seconds = (int)difftime( time(NULL), 
 1182                           info->work.starts.t_organize );
 1183 
 1184   BAF_OUT(PRIVATE);
 1185 }
 1186 
 1187 
 1188 /******************************************************************************
 1189 
 1190  FUNCTION: az_new_write_world
 1191 
 1192  SYNOPSIS
 1193  Writes the world information to the recall file
 1194 
 1195  HISTORY
 1196  Created  by dab
 1197 
 1198 ******************************************************************************/
 1199 
 1200 static AZ_RES
 1201 az_new_write_world(
 1202 AZ_NEW_INFO_PTR info
 1203 )
 1204 {
 1205   RECALL_BLOCK b_rc;
 1206   char *w_str = NULL;
 1207   int w_int = 0;
 1208   RECALL_CONTAINER w_cont = NULL;
 1209 
 1210   BAF_IN(PRIVATE);
 1211 
 1212   /* : use the world container created in the 'prep' */
 1213   w_cont = info->work.misc.w_cont;
 1214 
 1215   /* : world information */
 1216   /* : title */
 1217   w_str = info->params.world.title;
 1218   b_rc.size = strlen( w_str );
 1219   b_rc.type = 's';
 1220   b_rc.data.s = w_str;
 1221   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1222                 w_cont,
 1223                 "title",
 1224                   b_rc ) ) {
 1225     BAF_THROW_MSG("can't set world title" );
 1226   }
 1227 
 1228   /* : xdim */
 1229   w_int = info->results.organize.xdim;
 1230   b_rc.size = sizeof(int);
 1231   b_rc.type = 'd';
 1232   b_rc.data.d = w_int;
 1233   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1234                 w_cont,
 1235                 "xdim",
 1236                   b_rc ) ) {
 1237     BAF_THROW_MSG("can't set xdim" );
 1238   }
 1239 
 1240   /* : ydim */
 1241   w_int = info->results.organize.ydim;
 1242   b_rc.size = sizeof(int);
 1243   b_rc.type = 'd';
 1244   b_rc.data.d = w_int;
 1245   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1246                 w_cont,
 1247                 "ydim",
 1248                   b_rc ) ) {
 1249     BAF_THROW_MSG("can't set ydim" );
 1250   }
 1251 
 1252   /* : username */
 1253   w_str = info->params.origin.username;
 1254   b_rc.size = strlen( w_str );
 1255   b_rc.type = 's';
 1256   b_rc.data.s = w_str;
 1257   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1258                 w_cont,
 1259                 "username",
 1260                   b_rc ) ) {
 1261     BAF_THROW_MSG("can't set world username" );
 1262   }
 1263 
 1264   /* : TOBEDONE ... base date ... */
 1265 
 1266   /* : creation time */
 1267   w_int = (int)info->work.starts.t_start;
 1268   b_rc.size = sizeof(int);
 1269   b_rc.type = 'd';
 1270   b_rc.data.d = w_int;
 1271   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1272                 w_cont,
 1273                 "creation time",
 1274                   b_rc ) ) {
 1275     BAF_THROW_MSG("can't set creation time" );
 1276   }
 1277 
 1278   /* : modification time */
 1279   w_int = (int)time(NULL);
 1280   b_rc.size = sizeof(int);
 1281   b_rc.type = 'd';
 1282   b_rc.data.d = w_int;
 1283   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1284                 w_cont,
 1285                 "modification time",
 1286                   b_rc ) ) {
 1287     BAF_THROW_MSG("can't set modification time" );
 1288   }
 1289 
 1290   /* : user theme, default */
 1291   w_str = info->params.origin.user_theme;
 1292   b_rc.size = strlen( w_str );
 1293   b_rc.type = 's';
 1294   b_rc.data.s = w_str;
 1295   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1296                 w_cont,
 1297                 "user theme",
 1298                   b_rc ) ) {
 1299     BAF_THROW_MSG("can't set world user theme" );
 1300   }
 1301 
 1302   /* : system flag */
 1303   w_str = "new";
 1304   b_rc.size = strlen( w_str );
 1305   b_rc.type = 's';
 1306   b_rc.data.s = w_str;
 1307   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1308                   w_cont,
 1309                   "system flag",
 1310                   b_rc ) ) {
 1311     BAF_THROW_MSG("can't set world system flag" );
 1312   }
 1313 
 1314   BAF_OUT(PRIVATE);
 1315 }
 1316 
 1317 
 1318 /******************************************************************************
 1319 
 1320  FUNCTION: ac_new_write_origin
 1321 
 1322  SYNOPSIS
 1323  Writes the origin information to the recall file
 1324 
 1325  HISTORY
 1326  Created  by dab
 1327 
 1328 ******************************************************************************/
 1329 
 1330 static AZ_RES
 1331 ac_new_write_origin(
 1332 AZ_NEW_INFO_PTR info
 1333 )
 1334 {
 1335   RECALL_BLOCK b_rc;
 1336   char *w_str = NULL;
 1337   int w_int = 0;
 1338   RECALL_CONTAINER o_cont = NULL;
 1339 
 1340   BAF_IN(PRIVATE);
 1341 
 1342   /* : use the origin from the 'prep' call */
 1343   o_cont = info->work.misc.o_cont;
 1344 
 1345   /* : origin information */
 1346   /* : title */
 1347   w_str = info->params.origin.title;
 1348   b_rc.size = strlen( w_str );
 1349   b_rc.type = 's';
 1350   b_rc.data.s = w_str;
 1351   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1352                 o_cont,
 1353                 "title",
 1354                   b_rc ) ) {
 1355     BAF_THROW_MSG("can't origin title" );
 1356   }
 1357 
 1358   /* : url */
 1359   w_str = info->params.origin.base_url;
 1360   b_rc.size = strlen( w_str );
 1361   b_rc.type = 's';
 1362   b_rc.data.s = w_str;
 1363   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1364                 o_cont,
 1365                 "url",
 1366                   b_rc ) ) {
 1367     BAF_THROW_MSG("can't origin url" );
 1368   }
 1369 
 1370   /* : username */
 1371   w_str = info->params.origin.username;
 1372   b_rc.size = strlen( w_str );
 1373   b_rc.type = 's';
 1374   b_rc.data.s = w_str;
 1375   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1376                 o_cont,
 1377                 "username",
 1378                   b_rc ) ) {
 1379     BAF_THROW_MSG("can't set origin username" );
 1380   }
 1381 
 1382   /* : creation time */
 1383   w_int = (int)time(NULL);
 1384   b_rc.size = sizeof(int);
 1385   b_rc.type = 'd';
 1386   b_rc.data.d = w_int;
 1387   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1388                 o_cont,
 1389                 "creation time",
 1390                   b_rc ) ) {
 1391     BAF_THROW_MSG("can't set origin creation time" );
 1392   }
 1393 
 1394   /* : modification time */
 1395   w_int = (int)time(NULL);
 1396   b_rc.size = sizeof(int);
 1397   b_rc.type = 'd';
 1398   b_rc.data.d = w_int;
 1399   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1400                 o_cont,
 1401                 "modification time",
 1402                   b_rc ) ) {
 1403     BAF_THROW_MSG("can't set origin modification time" );
 1404   }
 1405 
 1406   /* : system flag */
 1407   w_str = "new";
 1408   b_rc.size = strlen( w_str );
 1409   b_rc.type = 's';
 1410   b_rc.data.s = w_str;
 1411   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1412                 o_cont,
 1413                 "system flag",
 1414                   b_rc ) ) {
 1415     BAF_THROW_MSG("can't set origin system flag" );
 1416   }
 1417 
 1418   /* : machine */
 1419   w_str = info->params.origin.machine;
 1420   b_rc.size = strlen( w_str );
 1421   b_rc.type = 's';
 1422   b_rc.data.s = w_str;
 1423   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1424                 o_cont,
 1425                 "machine",
 1426                   b_rc ) ) {
 1427     BAF_THROW_MSG("can't origin machine" );
 1428   }
 1429 
 1430   /* : generation time */
 1431   w_int = (int)difftime( time(NULL), info->work.starts.t_start );
 1432   b_rc.size = sizeof(int);
 1433   b_rc.type = 'd';
 1434   b_rc.data.d = w_int;
 1435   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1436                 o_cont,
 1437                 "seconds",
 1438                   b_rc ) ) {
 1439     BAF_THROW_MSG("can't set origin generation time" );
 1440   }
 1441 
 1442   /* : files */
 1443   w_int = info->results.totals.files;
 1444   b_rc.size = sizeof(int);
 1445   b_rc.type = 'd';
 1446   b_rc.data.d = w_int;
 1447   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1448                 o_cont,
 1449                 "files",
 1450                   b_rc ) ) {
 1451     BAF_THROW_MSG("can't set origin files" );
 1452   }
 1453 
 1454   /* : bytes */
 1455   w_int = info->results.totals.bytes;
 1456   b_rc.size = sizeof(int);
 1457   b_rc.type = 'd';
 1458   b_rc.data.d = w_int;
 1459   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1460                 o_cont,
 1461                 "bytes",
 1462                   b_rc ) ) {
 1463     BAF_THROW_MSG("can't set origin bytes" );
 1464   }
 1465 
 1466   BAF_OUT(PRIVATE);
 1467 }
 1468 
 1469 
 1470 /******************************************************************************
 1471 
 1472  FUNCTION: az_new_write_genesis
 1473 
 1474  SYNOPSIS
 1475  Writes pertinent recall file info before processing
 1476 
 1477  HISTORY
 1478  IncrDev Jun 14, 98 by dab: flexible enough for update
 1479  Created May 18, 98 by dab
 1480 
 1481 ******************************************************************************/
 1482 
 1483 static AZ_RES
 1484 az_new_write_genesis(
 1485 AZ_NEW_INFO_PTR info
 1486 )
 1487 {
 1488   RECALL_BLOCK b_rc;
 1489   void *w_buff = NULL;
 1490 #ifndef RECALL_SMALL
 1491   int w_buff_size = 0;
 1492 #endif
 1493   RECALL_CONTAINER g_cont = NULL;
 1494   RECALL_CONTAINER s_cont = NULL;
 1495   RECALL_CONTAINER p_cont = NULL;
 1496 
 1497   BAF_IN(PRIVATE);
 1498 
 1499   /* : use the origin from the 'prep' call */
 1500   g_cont = info->work.misc.g_cont;
 1501 
 1502   BAF_CATCH( recall_container_new(info->params.world.recall_file_ref,
 1503                   g_cont,
 1504                   "shared",
 1505                   &s_cont ) ) {
 1506     BAF_THROW_MSG("can't create shared container");
 1507   }
 1508                   
 1509   BAF_CATCH( recall_container_new(info->params.world.recall_file_ref,
 1510                   s_cont,
 1511                   "params",
 1512                   &p_cont ) ) {
 1513     BAF_THROW_MSG("can't create params container");
 1514   }
 1515                   
 1516 #ifndef RECALL_SMALL
 1517   /* : yes and no words */
 1518   /* : read the yes file into memory */
 1519   BAF_CATCH( 
 1520     util_file_read_file_into_memory( info->params.genesis.no_path,
 1521                      &w_buff_size,
 1522                      &w_buff ) ) {
 1523     BAF_THROW_MSG("can't read no file into memory" );
 1524   }
 1525   b_rc.size = w_buff_size;
 1526   b_rc.type = 't';
 1527   b_rc.data.t = w_buff;
 1528   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1529                   p_cont,
 1530                   "no words",
 1531                   b_rc ) ) {
 1532     GIVE(w_buff);
 1533     BAF_THROW_MSG("can't set no words" );
 1534   }
 1535   GIVE( w_buff );
 1536 
 1537   /* : yes words */
 1538   BAF_CATCH( 
 1539     util_file_read_file_into_memory( info->params.genesis.yes_path,
 1540                      &w_buff_size,
 1541                      &w_buff ) ) {
 1542     BAF_THROW_MSG("can't read no file into memory" );
 1543   }
 1544   b_rc.size = w_buff_size;
 1545   b_rc.type = 't';
 1546   b_rc.data.t = w_buff;
 1547   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1548                 p_cont,
 1549                 "yes words",
 1550                   b_rc ) ) {
 1551     GIVE( w_buff );
 1552     BAF_THROW_MSG("can't set yes words" );
 1553   }
 1554   GIVE( w_buff );
 1555 #endif
 1556 
 1557   /* : acquisition module */
 1558   b_rc.size = strlen( info->params.fetch.module );
 1559   b_rc.type = 's';
 1560   b_rc.data.s = info->params.fetch.module;
 1561   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1562                   p_cont,
 1563                   "fetch module",
 1564                   b_rc ) ) {
 1565     BAF_THROW_MSG("can't set fetch module" );
 1566   }
 1567 
 1568   /* : get module */
 1569   b_rc.size = strlen( info->params.fetch.get_module );
 1570   b_rc.type = 's';
 1571   b_rc.data.s = info->params.fetch.get_module;
 1572   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1573                   p_cont,
 1574                   "get module",
 1575                   b_rc ) ) {
 1576     BAF_THROW_MSG("can't set get module" );
 1577   }
 1578 
 1579   /* : get action */
 1580   b_rc.size = strlen( info->params.fetch.get_action );
 1581   b_rc.type = 's';
 1582   b_rc.data.s = info->params.fetch.get_action;
 1583   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1584                   p_cont,
 1585                   "get action",
 1586                   b_rc ) ) {
 1587     BAF_THROW_MSG("can't set get action" );
 1588   }
 1589 
 1590   /* : file extensions */
 1591   b_rc.size = strlen( info->params.fetch.file_extensions );
 1592   b_rc.type = 's';
 1593   b_rc.data.s = info->params.fetch.file_extensions;
 1594   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1595                   p_cont,
 1596                   "file extensions",
 1597                   b_rc ) ) {
 1598     BAF_THROW_MSG("can't set file extensions" );
 1599   }
 1600 
 1601   b_rc.size = sizeof(int);
 1602   b_rc.type = 'd';
 1603   b_rc.data.d = info->params.fetch.recursion_level;
 1604   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1605                   p_cont,
 1606                   "recursion level",
 1607                   b_rc ) ) {
 1608     BAF_THROW_MSG("can't set recursion level" );
 1609   }
 1610 
 1611   /* : exclude domains */
 1612   b_rc.size = strlen( info->params.fetch.exclude_domains );
 1613   b_rc.type = 's';
 1614   b_rc.data.s = info->params.fetch.exclude_domains;
 1615   BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1616                   p_cont,
 1617                   "exclude domains",
 1618                   b_rc ) ) {
 1619     BAF_THROW_MSG("can't set exclude domains" );
 1620   }
 1621 
 1622   /* : search information, if searching */
 1623   if ( info->params.search.url != NULL ) {
 1624 
 1625     /* : the search url, don't need the file, it's temp */
 1626     b_rc.size = strlen( info->params.search.url );
 1627     b_rc.type = 's';
 1628     b_rc.data.s = info->params.search.url;
 1629     BAF_CATCH( recall_property_set( info->params.world.recall_file_ref,
 1630                 p_cont,
 1631                 "search url",
 1632                 b_rc ) ) {
 1633       BAF_THROW_MSG("can't set search url");
 1634     }
 1635 
 1636   }
 1637 
 1638   GIVE( w_buff );
 1639 
 1640   BAF_OUT(PRIVATE);
 1641 }
 1642 
 1643 
 1644 #if 0
 1645 /******************************************************************************
 1646 
 1647  SYNOPSIS
 1648  For testing the org module without acquire or comprehension
 1649 
 1650 ******************************************************************************/
 1651 
 1652 static AZ_RES
 1653 test_load_dna(
 1654 AZ_NEW_INFO_PTR info
 1655 )
 1656 {
 1657   BAF_CATCH(
 1658     util_file_read_file_into_memory( info->work.paths.dna,
 1659                      (size_t *)&(info->work.dna.size),
 1660                      (void *)&(info->work.dna.buff) ) ) {
 1661     BAF_THROW_MSG("can't read dna into memory");
 1662   }
 1663 
 1664   /* read the x and y dims */
 1665   sscanf( info->work.dna.buff, "%*d %*s %d %d %*s\n",
 1666       &(info->results.comprehend.xdim), 
 1667       &(info->results.comprehend.ydim) );
 1668 
 1669   MSG(PRIVATE,"TEMP: *** %s\n", "" );
 1670   MSG(PRIVATE,"TEMP: *** %s\n", "... fudging a dna from existing file" );
 1671   MSG(PRIVATE,
 1672       "TEMP: *** dna_xdim scanned: %d\n", info->results.comprehend.xdim );
 1673   MSG(PRIVATE,
 1674       "TEMP: *** dna_ydim scanned: %d\n", info->results.comprehend.ydim );
 1675   MSG(PRIVATE,"TEMP: *** %s\n", "" );
 1676 
 1677   BAF_OUT(PRIVATE);
 1678 }
 1679 #endif
 1680 
 1681 /******************************************************************************
 1682 
 1683  FUNCTION: az_new_progress
 1684 
 1685  SYNOPSIS
 1686  Sets the session progress for the new action
 1687 
 1688  HISTORY
 1689  Created  by dab
 1690 
 1691 ******************************************************************************/
 1692 
 1693 static AZ_RES
 1694 az_new_progress(
 1695 AZ_NEW_INFO_PTR info,
 1696 char *caption,
 1697 int increment,
 1698 int total
 1699 )
 1700 {
 1701   char format_str[] = "Creating new world titled \"%s\".\n\n%s";
 1702   ENDO_SESSION_STATUS_BLOCK b_sess;
 1703   char *status_str = NULL;
 1704 
 1705   BAF_IN(PRIVATE);
 1706 
 1707   TAKE( status_str, (char *), strlen(format_str) + 
 1708     strlen(info->params.world.title) + strlen(caption) + 1 ) {
 1709     BAF_ERROR();
 1710   }
 1711   sprintf( status_str, format_str, info->params.world.title,
 1712        caption );
 1713 
 1714   b_sess.total = total;
 1715   b_sess.amount = increment;
 1716   b_sess.str = status_str;
 1717   b_sess.user_data = NULL;
 1718   BAF_CATCH( endo_msg_session_status_set( info->params.BAF.msg_ref, b_sess ) ) {
 1719     BAF_ERROR();
 1720   }
 1721 
 1722   GIVE(status_str);
 1723   BAF_OUT(PRIVATE);
 1724 
 1725  BAF_ERROR:
 1726 
 1727   GIVE(status_str);
 1728   BAF_THROW();
 1729 }
 1730 
 1731 
 1732 /******************************************************************************
 1733 
 1734  FUNCTION: az_write_genome
 1735 
 1736  SYNOPSIS
 1737  Writes a genome to file by sending save message to rd_dna module
 1738 
 1739 ******************************************************************************/
 1740 
 1741 static AZ_RES
 1742 az_write_genome(
 1743 AZ_NEW_INFO_PTR info
 1744 )
 1745 {
 1746   ENDO_MSG_REF response = NULL;
 1747   BAF_BOOLEAN is_success = BAF_FALSE;
 1748 
 1749   BAF_IN(PRIVATE);
 1750 
 1751   BAF_CATCH( endo_msg_send_simple( NULL,
 1752                    AZ_MY_NAME,
 1753                    "rd_dna",
 1754                    "save",
 1755                    ENDO_RESPONSE_WAIT,
 1756                    &response,
 1757                    "pdss",
 1758                    info->work.dna.buff,
 1759                    info->work.dna.size,
 1760                    info->params.world.recall_file,
 1761                    "genome" )) {
 1762     BAF_THROW_MSG("can't send");
 1763   }
 1764 
 1765   BAF_CATCH( endo_msg_response_is_success( response, &is_success ) ) {
 1766     BAF_THROW_MSG("can't determine success or fail");
 1767   }
 1768 
 1769   if ( is_success == BAF_FALSE ) {
 1770     BAF_THROW();
 1771   }
 1772 
 1773   BAF_OUT(PRIVATE);
 1774 }
 1775 
 1776 
 1777 /******************************************************************************
 1778 
 1779  FUNCTION: az_load_params_for_new
 1780 
 1781  SYNOPSIS
 1782  Loads the params for the 'new' message into the info struct
 1783 
 1784 ******************************************************************************/
 1785 
 1786 static AZ_RES
 1787 az_load_params_for_new(
 1788 ENDO_MSG_REF msg,
 1789 AZ_NEW_INFO_PTR info
 1790 )
 1791 {
 1792   BAF_IN(PRIVATE);
 1793 
 1794   /* : get our args */
 1795   BAF_CATCH( 
 1796     endo_msg_get_args( msg,
 1797                "sssssssssssssssssdssdfddsffddsfddd",
 1798                &(info->params.world.recall_file),
 1799                &(info->params.world.title),
 1800                &(info->params.world.temp_path),
 1801                &(info->params.genesis.no_path),
 1802                &(info->params.genesis.yes_path),
 1803                &(info->params.origin.title),
 1804                &(info->params.origin.machine),
 1805                &(info->params.origin.base_url),
 1806                &(info->params.origin.username),
 1807                &(info->params.origin.user_theme),
 1808                &(info->params.origin.fetch_path),
 1809                &(info->params.fetch.module),
 1810                &(info->params.fetch.http_proxy),
 1811                &(info->params.fetch.ftp_proxy),
 1812                &(info->params.fetch.get_module),
 1813                &(info->params.fetch.get_action),
 1814                &(info->params.fetch.file_extensions),
 1815                &(info->params.fetch.recursion_level),
 1816                &(info->params.fetch.exclude_domains),
 1817                &(info->params.info.endings_and_cleaners),
 1818                &(info->params.comprehend.num_keys_per_file),
 1819                &(info->params.comprehend.target_q_error),
 1820                &(info->params.comprehend.max_training_runs),
 1821                &(info->params.comprehend.max_training_seconds),
 1822                &(info->params.comprehend.params_mod),
 1823                &(info->params.comprehend.m_density),
 1824                &(info->params.organize.target_q_error),
 1825                &(info->params.organize.max_training_runs),
 1826                &(info->params.organize.max_training_seconds),
 1827                &(info->params.organize.params_mod),
 1828                &(info->params.organize.m_density),
 1829                &(info->params.organize.umat_average),
 1830                &(info->params.organize.umat_median),
 1831                &(info->params.organize.fp_reduction_factor) ) ) {
 1832 
 1833     BAF_THROW_MSG("can't get args for new");
 1834   }
 1835 
 1836   BAF_OUT(PRIVATE);
 1837 }
 1838 
 1839 
 1840 /******************************************************************************
 1841 
 1842  FUNCTION: az_verify_base_url_for_search
 1843 
 1844  SYNOPSIS
 1845  For convenience, checks and makes if required the file-url for our search
 1846 
 1847 ******************************************************************************/
 1848 
 1849 static AZ_RES
 1850 az_verify_base_url_for_search(
 1851 AZ_NEW_INFO_PTR info
 1852 )
 1853 {
 1854   char *prospective_url = NULL;
 1855 
 1856   BAF_IN(PRIVATE);
 1857 
 1858   TAKE( prospective_url, (char *), 
 1859     (strlen( info->params.search.file ) + strlen("file://") + 1) ) {
 1860     BAF_THROW_MSG("can't allocate");
 1861   }
 1862   sprintf( prospective_url,"%s%s",
 1863        "file://", info->params.search.file );
 1864 
 1865   /* : if the base url passed in doesn't match, over-ride it */
 1866   if ( strcmp( prospective_url, info->params.origin.base_url ) != 0 ) {
 1867 
 1868     GIVE( info->params.origin.base_url );
 1869     info->params.origin.base_url = prospective_url;
 1870 
 1871   }
 1872   else {
 1873 
 1874     GIVE( prospective_url );
 1875 
 1876   }
 1877 
 1878   BAF_OUT(PRIVATE);
 1879 }
 1880 
 1881 
 1882 /******************************************************************************
 1883 
 1884  FUNCTION: az_load_params_for_new_search
 1885 
 1886  SYNOPSIS
 1887  Loads the params for the 'new_search' message into the info struct
 1888 
 1889 ******************************************************************************/
 1890 
 1891 static AZ_RES
 1892 az_load_params_for_new_search(
 1893 ENDO_MSG_REF msg,
 1894 AZ_NEW_INFO_PTR info
 1895 )
 1896 {
 1897   BAF_IN(PRIVATE);
 1898 
 1899   /* : get our args */
 1900   BAF_CATCH( 
 1901     endo_msg_get_args( msg,
 1902                "sssssssssssssssssdssdfddsffddsfdddss",
 1903                &(info->params.world.recall_file),
 1904                &(info->params.world.title),
 1905                &(info->params.world.temp_path),
 1906                &(info->params.genesis.no_path),
 1907                &(info->params.genesis.yes_path),
 1908                &(info->params.origin.title),
 1909                &(info->params.origin.machine),
 1910                &(info->params.origin.base_url),
 1911                &(info->params.origin.username),
 1912                &(info->params.origin.user_theme),
 1913                &(info->params.origin.fetch_path),
 1914                &(info->params.fetch.module),
 1915                &(info->params.fetch.http_proxy),
 1916                &(info->params.fetch.ftp_proxy),
 1917                &(info->params.fetch.get_module),
 1918                &(info->params.fetch.get_action),
 1919                &(info->params.fetch.file_extensions),
 1920                &(info->params.fetch.recursion_level),
 1921                &(info->params.fetch.exclude_domains),
 1922                &(info->params.info.endings_and_cleaners),
 1923                &(info->params.comprehend.num_keys_per_file),
 1924                &(info->params.comprehend.target_q_error),
 1925                &(info->params.comprehend.max_training_runs),
 1926                &(info->params.comprehend.max_training_seconds),
 1927                &(info->params.comprehend.params_mod),
 1928                &(info->params.comprehend.m_density),
 1929                &(info->params.organize.target_q_error),
 1930                &(info->params.organize.max_training_runs),
 1931                &(info->params.organize.max_training_seconds),
 1932                &(info->params.organize.params_mod),
 1933                &(info->params.organize.m_density),
 1934                &(info->params.organize.umat_average),
 1935                &(info->params.organize.umat_median),
 1936                &(info->params.organize.fp_reduction_factor),
 1937                &(info->params.search.url),
 1938                &(info->params.search.file) ) ) {
 1939 
 1940     BAF_THROW_MSG("can't get args for new");
 1941   }
 1942 
 1943   /* : get the file url */
 1944   BAF_CATCH( az_verify_base_url_for_search( info ) ) {
 1945     BAF_THROW_MSG("can't make url for search file");
 1946   }
 1947 
 1948   BAF_OUT(PRIVATE);
 1949 }
 1950 
 1951 
 1952 /******************************************************************************
 1953 
 1954  FUNCTION: az_new_print_telemetry
 1955 
 1956  SYNOPSIS
 1957  Spill some telemetry about a 'new' action
 1958 
 1959 ******************************************************************************/
 1960 
 1961 static AZ_RES
 1962 az_new_print_telemetry(
 1963 AZ_NEW_INFO_PTR info
 1964 )
 1965 {
 1966   BAF_IN(PRIVATE);
 1967 
 1968   MSG(PRIVATE,"total files: %d", info->results.totals.files );
 1969   MSG(PRIVATE,"total bytes: %d", info->results.totals.bytes );
 1970   MSG(PRIVATE,"total seconds: %d", info->results.totals.seconds );
 1971   MSG(PRIVATE,"fetched: %d", info->results.fetch.files_fetched );
 1972   MSG(PRIVATE,"acquired: %d", info->results.fetch.files_acquired );
 1973   MSG(PRIVATE,"fetch seconds: %d", info->results.fetch.seconds );
 1974   MSG(PRIVATE,"infoed: %d", info->results.info.files );
 1975   MSG(PRIVATE,"infoed seconds: %d", info->results.info.seconds );
 1976   MSG(PRIVATE,"concepts: %d", info->results.comprehend.concepts );
 1977   MSG(PRIVATE,"contexts: %d", info->results.comprehend.contexts );
 1978   MSG(PRIVATE,"comprehension seconds %d", info->results.comprehend.seconds );
 1979   MSG(PRIVATE,"comprehension training: %d", 
 1980       info->results.comprehend.training_seconds );
 1981   MSG(PRIVATE,"comprehension q_error: %f", info->results.comprehend.q_error );
 1982   MSG(PRIVATE,"fingerprinter xdim: %d", info->results.comprehend.xdim );
 1983   MSG(PRIVATE,"fingerprinter ydim: %d", info->results.comprehend.ydim );
 1984   MSG(PRIVATE,"organization labels: %d", info->results.organize.labels );
 1985   MSG(PRIVATE,"organization docs: %d", info->results.organize.docs );
 1986   MSG(PRIVATE,"organization seconds: %d", info->results.organize.seconds );
 1987   MSG(PRIVATE,"organization training seconds: %d", 
 1988       info->results.organize.training_seconds );
 1989   MSG(PRIVATE,"organization q_error: %f", info->results.organize.q_error );
 1990   MSG(PRIVATE,"plane xdim: %d", info->results.organize.xdim );
 1991   MSG(PRIVATE,"plane ydim: %d", info->results.organize.ydim );
 1992   MSG(PRIVATE,"analysis DONE!");
 1993 
 1994   BAF_OUT(PRIVATE);
 1995 }
 1996 
 1997 
 1998 #if 0
 1999 /******************************************************************************
 2000 
 2001  FUNCTION: az_new_search_do
 2002 
 2003  SYNOPSIS
 2004  Actually carries out the search by fetching the search url to a file
 2005 
 2006 ******************************************************************************/
 2007 
 2008 static AZ_RES
 2009 az_new_search_do(
 2010 AZ_NEW_INFO_PTR info
 2011 )
 2012 {
 2013   char *file_name = NULL;
 2014   char *full_proxy = NULL;
 2015   char *url = NULL;
 2016   char *proxy = NULL;
 2017   unsigned short port = 0;
 2018   FILE *file = NULL;
 2019   long conf = CONF_PORT | CONF_PROXY;
 2020 
 2021   BAF_IN(PRIVATE);
 2022 
 2023   /* : for convenience, no disposal required */
 2024   full_proxy = info->params.fetch.http_proxy;
 2025   file_name = info->params.search.file;
 2026   url = info->params.search.url;
 2027 
 2028   /* : get a work buffer; disposal required */
 2029   TAKE( proxy, (char *), 1024 ) {
 2030     BAF_ERROR_MSG("can't allocate");
 2031   }
 2032 
 2033   /* : see if the proxy is blank */
 2034   if ( strstr(full_proxy,":") == NULL ) {
 2035     conf = 0;
 2036   }
 2037   else {
 2038 
 2039     sscanf(full_proxy,"%1023[^\n:]:%hu", proxy, &port);
 2040 
 2041   }
 2042 
 2043   /* : open the file write-mode; don't forget to close */
 2044   file = fopen(file_name, "w");
 2045   if ( file == NULL ) {
 2046     BAF_ERROR_MSG("can't open dest file");
 2047   }
 2048 
 2049   BAF_CATCH( urlget( file, url, port, proxy, conf ) ) {
 2050     BAF_ERROR_MSG("can't get url");
 2051   }
 2052 
 2053   if ( file != NULL ) 
 2054     fclose( file );
 2055   GIVE(proxy);
 2056 
 2057   BAF_OUT(PRIVATE);
 2058 
 2059 
 2060  BAF_ERROR:
 2061 
 2062   if ( file != NULL ) 
 2063     fclose( file );
 2064   GIVE(proxy);
 2065 
 2066   BAF_THROW();
 2067 }
 2068 #endif
 2069 
 2070 /******************************************************************************
 2071 
 2072  FUNCTION: az_handle_new
 2073 
 2074  SYNOPSIS
 2075  Handles the new message
 2076 
 2077  HISTORY
 2078  Created May 17, 98 by dab
 2079 
 2080 ******************************************************************************/
 2081 
 2082 static AZ_RES
 2083 az_handle_new(
 2084 ENDO_MSG_REF msg,
 2085 BAF_BOOLEAN search,
 2086 char **pp_sub_err_str
 2087 )
 2088 {
 2089   AZ_RES az_res = 0;
 2090   AZ_NEW_INFO_PTR info = NULL;
 2091   int increment = 0;
 2092   int total = 0;
 2093   char *quick_mode = "quick";
 2094   char *normal_mode = "normal";
 2095   char *info_mode = NULL;
 2096 
 2097   BAF_IN(PRIVATE);
 2098 
 2099   /* : we'll need an info structure */
 2100   BAF_CATCH( az_new_new_info( &info ) ) {
 2101     az_res = AZ_CANT_ALLOCATE_INFO;
 2102     BAF_ERROR();
 2103   }
 2104 
 2105   info->params.BAF.msg_ref = msg;
 2106 
 2107   /* : start the clock */
 2108   info->work.starts.t_start = time(NULL);
 2109 
 2110   /* : get the args */
 2111   if ( search == BAF_TRUE ) {
 2112 
 2113     /* ... disabled the fetching of the url to file
 2114     total = 6;
 2115     */
 2116     total = 5;
 2117     BAF_CATCH( az_load_params_for_new_search( msg, info ) ) {
 2118       az_res = AZ_BAD_PARAMS;
 2119       BAF_ERROR();
 2120     }
 2121 
 2122     /* ... disabled the fetching of the url to file
 2123     BAF_CATCH( az_new_progress( info, "Searching ...",
 2124                 increment, total ) ) {
 2125       MSG(PRIVATE,"can't set progress");
 2126     }
 2127     increment++;
 2128 
 2129     BAF_CATCH( az_new_search_do( info ) ) {
 2130       az_res = AZ_CANT_SEARCH;
 2131       BAF_ERROR();
 2132     }
 2133     */
 2134 
 2135     info_mode = quick_mode;
 2136 
 2137   }
 2138   else {
 2139 
 2140     total = 5;
 2141     BAF_CATCH( az_load_params_for_new( msg, info ) ) {
 2142       az_res = AZ_BAD_PARAMS;
 2143       BAF_ERROR();
 2144     }
 2145 
 2146     info_mode = normal_mode;
 2147 
 2148   }
 2149 
 2150   BAF_CATCH( az_new_progress( info, "Starting ...",
 2151               increment, total ) ) {
 2152     MSG(PRIVATE,"can't set progress");
 2153   }
 2154   increment++;
 2155 
 2156   /* : prep */
 2157   BAF_CATCH( az_new_prep( info ) ) {
 2158     az_res = AZ_CANT_PREP;
 2159     BAF_ERROR();
 2160   }
 2161 
 2162   BAF_CATCH( az_new_progress( info, "Fetching ...",
 2163               increment, total ) ) {
 2164     MSG(PRIVATE,"can't set progress");
 2165   }
 2166   increment++;
 2167 
 2168   /* : fetch an origin, takes acquire or update */
 2169   BAF_CATCH( az_new_fetch( info ) ) {
 2170     az_res = AZ_CANT_ACQUIRE;
 2171     BAF_ERROR();
 2172   }
 2173 
 2174   /* : check here if we've fetched any files */
 2175   if ( info->results.fetch.files_acquired < 1 ) {
 2176     az_res = AZ_NO_DOCUMENTS_FOUND;
 2177     BAF_ERROR();
 2178   }
 2179 
 2180   BAF_CATCH( az_new_progress( info, "Infoing ...",
 2181               increment, total ) ) {
 2182     MSG(PRIVATE,"can't set progress");
 2183   }
 2184 
 2185   /* : info the origin, takes info or update */
 2186   BAF_CATCH( az_new_info( info, info_mode ) ) {
 2187     az_res = AZ_CANT_INFO;
 2188     BAF_ERROR();
 2189   }
 2190 
 2191   BAF_CATCH( az_new_progress( info, "Comprehending ...",
 2192               increment, total ) ) {
 2193     MSG(PRIVATE,"can't set progress");
 2194   }
 2195   increment++;
 2196 
 2197   /* : comprehend the origin, takes only comprehend */
 2198   BAF_CATCH( az_new_comprehend( info ) ) {
 2199     az_res = AZ_CANT_COMPREHEND;
 2200     BAF_ERROR();
 2201   }
 2202 
 2203   /* : take a moment to write the genome to the recall file */
 2204   BAF_CATCH( az_write_genome( info ) ) {
 2205     az_res = AZ_CANT_WRITE_GENOME;
 2206     BAF_ERROR();
 2207   }
 2208 
 2209   BAF_CATCH( az_new_progress( info, "Organizing ...",
 2210               increment, total ) ) {
 2211     MSG(PRIVATE,"can't set progress");
 2212   }
 2213   increment++;
 2214 
 2215   /* : organize the origin, update a separate function */
 2216   BAF_CATCH( az_new_organize( info ) ) {
 2217     az_res = AZ_CANT_ORGANIZE;
 2218     BAF_ERROR();
 2219   }
 2220 
 2221   /* : stop the clock */
 2222   info->results.totals.seconds = (int)difftime( time(NULL),
 2223                        info->work.starts.t_start );
 2224 
 2225   BAF_CATCH( az_new_progress( info, "Finishing ...",
 2226               increment, total ) ) {
 2227     MSG(PRIVATE,"can't set progress");
 2228   }
 2229   increment++;
 2230 
 2231   /* : write the goodies to the recall file */
 2232   BAF_CATCH( az_new_write_world( info ) ) {
 2233     az_res = AZ_CANT_WRITE_RECALL;
 2234     BAF_ERROR();
 2235   }
 2236 
 2237   BAF_CATCH( ac_new_write_origin( info ) ) {
 2238     az_res = AZ_CANT_WRITE_RECALL;
 2239     BAF_ERROR();
 2240   }
 2241 
 2242   BAF_CATCH( az_new_write_genesis( info ) ) {
 2243     az_res = AZ_CANT_WRITE_RECALL;
 2244     BAF_ERROR();
 2245   }
 2246 
 2247   /* : print some telemetry */
 2248   BAF_CATCH( az_new_print_telemetry( info ) ) {
 2249     MSG(PRIVATE,"can't print telemetry");
 2250   }
 2251 
 2252   /* : must save the recall file before replying */
 2253   BAF_CATCH( recall_file_save( info->params.world.recall_file_ref ) ) {
 2254     MSG(PRIVATE,"can't save recall file" );
 2255   }
 2256   
 2257   /* : successful reply handled here */
 2258   BAF_CATCH( endo_msg_reply_args( msg,
 2259                   "ssddddddddddddfddddddfdd",
 2260                   info->params.world.recall_file,
 2261                   info->params.world.title,
 2262                   info->results.totals.files,
 2263                   info->results.totals.bytes,
 2264                   info->results.totals.seconds,
 2265                   info->results.fetch.files_fetched,
 2266                   info->results.fetch.files_acquired,
 2267                   info->results.fetch.seconds,
 2268                   info->results.info.files,
 2269                   info->results.info.seconds,
 2270                   info->results.comprehend.concepts,
 2271                   info->results.comprehend.contexts,
 2272                   info->results.comprehend.seconds,
 2273                   info->results.comprehend.training_seconds,
 2274                   info->results.comprehend.q_error,
 2275                   info->results.comprehend.xdim,
 2276                   info->results.comprehend.ydim,
 2277                   info->results.organize.labels,
 2278                   info->results.organize.docs,
 2279                   info->results.organize.seconds,
 2280                   info->results.organize.training_seconds,
 2281                   info->results.organize.q_error,
 2282                   info->results.organize.xdim,
 2283                   info->results.organize.ydim ) ) {
 2284     az_res = AZ_CANT_REPLY;
 2285     BAF_ERROR();
 2286   }
 2287 
 2288   /* : clean up */
 2289   if ( info != NULL ) {
 2290     BAF_CATCH( az_new_dispose_info( &info ) ) {
 2291       BAF_ERROR_MSG("can't dispose info");
 2292     }
 2293   }
 2294 
 2295   BAF_OUT(PRIVATE);
 2296 
 2297 
 2298  BAF_ERROR:
 2299 
 2300   if ( az_res == AZ_OK ) 
 2301     az_res = AZ_FAIL;
 2302 
 2303   /* : get the sub error from an underlying module if any */
 2304   /* : will indicate cancelation */
 2305   if ( info->work.error.str != NULL ) {
 2306     TAKE( *pp_sub_err_str ,(char *), strlen(info->work.error.str) + 1 ) {
 2307       *pp_sub_err_str = NULL;
 2308     } else {
 2309       strcpy( *pp_sub_err_str, info->work.error.str );
 2310     }
 2311   }
 2312 
 2313   /* : if there is a recall file, then delete it */
 2314   if ( info->params.world.recall_file != NULL ) {
 2315 
 2316     /* : make sure the recall sub-system unloads it */
 2317     if ( info->params.world.recall_file_ref != NULL ) {
 2318 
 2319       BAF_CATCH( recall_file_close( &(info->params.world.recall_file_ref) ) ) {
 2320     MSG(PRIVATE,"can't close recall file" );
 2321       }
 2322       
 2323     }
 2324 
 2325     remove( info->params.world.recall_file );
 2326 
 2327   }
 2328 
 2329   if ( info != NULL ) {
 2330     BAF_CATCH( az_new_dispose_info( &info ) ) {
 2331       MSG(PRIVATE,"can't dispose info" );
 2332     }
 2333   }
 2334 
 2335   BAF_THROW_ERR( az_res );
 2336 }
 2337 
 2338 
 2339 /******************************************************************************
 2340 
 2341  FUNCTION: analyze
 2342 
 2343  SYNOPSIS
 2344  Main entry point for this module
 2345 
 2346  HISTORY
 2347  Created May 17, 98 by dab
 2348 
 2349  NOTE
 2350  Because of the volume of reply arguments, we'll reply inside the handlers.
 2351 
 2352 ******************************************************************************/
 2353 
 2354 int
 2355 analyze(
 2356 ENDO_MSG_REF msg_ref
 2357 )
 2358 {
 2359   char *action = NULL;
 2360   AZ_RES az_res = 0;
 2361   char *sub_err_str = NULL;
 2362   char *combined_err_str = NULL;
 2363 
 2364   BAF_IN(PUBLIC);
 2365 
 2366   BAF_CATCH( endo_msg_get_action( msg_ref, &action ) ) {
 2367     BAF_ERROR();
 2368   }
 2369 
 2370   /* : for 'new' */
 2371   if ( strcmp( AZ_NEW, action ) == 0 ) {
 2372 
 2373     /* : call our handler for this case, replies internally */
 2374     BAF_CATCH_ERR( az_res, az_handle_new( msg_ref, BAF_FALSE, &sub_err_str ) ) {
 2375       BAF_ERROR();
 2376     }
 2377 
 2378     /* : we can go directly to good exit ... */
 2379     goto SKIP_REPLY;
 2380 
 2381   }
 2382 
 2383   else if ( strcmp( AZ_NEW_SEARCH, action ) == 0 ) {
 2384 
 2385     /* : as above */
 2386     BAF_CATCH_ERR( az_res, az_handle_new( msg_ref, BAF_TRUE, &sub_err_str ) ) {
 2387       BAF_ERROR();
 2388     }
 2389 
 2390     goto SKIP_REPLY;
 2391 
 2392   }
 2393 
 2394   else {
 2395 
 2396     /* : it's an unsuported action type */
 2397     az_res = AZ_UNSUPPORTED;
 2398     BAF_ERROR();
 2399 
 2400   }
 2401 
 2402   BAF_CATCH( endo_msg_reply_success( msg_ref ) ) {
 2403     az_res = AZ_CANT_REPLY;
 2404     BAF_ERROR();
 2405   }
 2406 
 2407 
 2408  SKIP_REPLY:
 2409 
 2410   GIVE( action );
 2411 
 2412   BAF_OUT(PUBLIC);
 2413 
 2414 
 2415  BAF_ERROR:
 2416 
 2417   /* : make sure that we have an error */
 2418   if ( az_res == AZ_OK )
 2419     az_res = AZ_FAIL;
 2420 
 2421   /* : if we have a sub_err_str, combine it with ours and return it */
 2422   if ( sub_err_str != NULL ) {
 2423 
 2424     TAKE( combined_err_str ,(char *), (strlen(sub_err_str) +
 2425       strlen(s_error[az_res]) +
 2426       10 ) ) {
 2427 
 2428       /* : standard reply */
 2429       BAF_CATCH( endo_msg_reply_err( msg_ref, s_error[az_res] ) ) {
 2430     MSG(PRIVATE,"can't reply" );
 2431       }
 2432 
 2433       MSG(PRIVATE,s_error[az_res]);
 2434 
 2435     } else {
 2436 
 2437       sprintf( combined_err_str, "%s < %s >", s_error[az_res], sub_err_str );
 2438       BAF_CATCH( endo_msg_reply_err( msg_ref, combined_err_str) ) {
 2439     MSG(PRIVATE,"can't reply" );
 2440       }
 2441 
 2442       MSG(PRIVATE,combined_err_str);
 2443     }
 2444 
 2445   }
 2446   else {
 2447 
 2448     /* : standard reply */
 2449     BAF_CATCH( endo_msg_reply_err( msg_ref, s_error[az_res] ) ) {
 2450       MSG(PRIVATE,"can't reply" );
 2451     }
 2452 
 2453     MSG(PRIVATE,s_error[az_res]);
 2454   }
 2455 
 2456   /* : clean up */
 2457   GIVE(sub_err_str);
 2458   GIVE(combined_err_str);
 2459   GIVE( action );
 2460 
 2461   BAF_THROW_MSG( s_error[az_res] );
 2462 }
 2463 
 2464