KazMuzik.net

Music / Technology / Immigration / アメリカ

145303 <- 205797 205872 206190 206548 206825 207062 207320 207493 207679 207967 208347 208527 208712 209045 209208 209509 209845 210114 210264 210456 210825 211054 211268 211494 211799 212195 212373 212731 212843 213073 213444 213707 213801 214104 214389 214736 214893 215047 215315 215731 215896 216117 216402 216649 216851 217187 217415 217728 217981 218151 218420 218709 219104 219369 219446 219810 220035 220198 220575 220737 221027 221433 221541 221736 221961 222220 222634 222901 223216 223430 223586 223981 224060 224306 224532 224991 225133 225388 225644 225982 226153 226485 226617 226993 227201 227374 227695 227993 228312 228505 228744 229048 229180 229579 229743 230024 230164 230637 230728 230963 231309 231574 231695 232129 232338 232677 232828 232990 233467 233632 233937 234034 234368 234695 234878 235220 235443 235570 235811 236189 236454 236592 236948 237064 237459 237724 237905 238245 238339 238780 238890 239183 239368 239642 239888 240213 240581 240649 240923 241334 241577 241773 241997 242330 242620 242890 243158 243211 243506 243718 243969 244328 244599 244918 245219 245345 245506 245909 246163 246503 246649 246911 247204 247344 247683 247889 248127 248344 248751 248898 249161 249526 249674 249934 250271 250529 250906 251272 250653 251489 251840 251951 252405 252671 252771 253029 253243 253629 253820 254021 254251 254480 254946 255198 255304 255640 255949 256078 256502 256703 256845 257525 257544 257900 257251 258440 258634 258182 258941 259137 259519 259828 259950 260212 260494 260839 261095 261166 261588 261883 262066 262188 262614 262783 262921 263237 263622 263725 264181 264363 264658 264712 265221 265167 265536 265925 266110 266295 266549 266967 267127 267328 267647 267861 268736 268895 269191 268120 269357 268352 269692 269950 270110 270403 270783 271091 271270 271586 271847 272056 272253 272468 272697 272960 273285 273630 273870 273962 274185 274647 274892 275049 275352 275678 275836 276048 276338 276735 276894 277104 277413 277535 277946 278133 278287 278772 278925 279261 279364 279561 280046 280185 280341 280590 280979 281294 281502 281829 281953 282338 282622 282655 283151 283102 283565 283776 284137 284220 284727 285077 284626 285191 285656 285936 286066 286281 286566 286959 287091 287327 287491 287761 288043 288396 288734 288966 289104 289456 289641 290029 290293 290438 290775 291029 291096 291348 291706 291958 292148 292606 292672 293035 293173 293581 293678 293995 294277 294433 294904 294944 295198 295531 295688 295997 296414 296580 296931 297056 297332 297661 297899 297991 298462 298686 298777 299075 299453 299586 300028 300113 300486 300757 301029 301444 301676 301952 302286 302621 302501 301184 302896 303252 303589 303792 304046 304149 304544 304804 305572 305114 305269 305778 306118 306402 306496 306769 306984 307302 307688 307964 308195 308344 308546 308841 309232 309314 309711 309854 310041 310417 310567 310959 311092 311382 311576 311970 312300 312491 312752 313010 313181 313538 313735 313969 314318 314376 314692 315097 315235 315430 315743 315996 316411 316610 316803 316951 317267 317489 317737 318044 318447 318562 318767 319216 319292 319520 319908 320021 320329 320709 320902 321236 321392 321568 322004 322079 322341 322682 322961 323152 323471 323783 323956 324204 324582 324828 324945 325267 325563 325791 325930 326237 326655 326805 327140 327360 327669 327925 328132 328223 328489 328819 329120 329442 329486 329825 330126 330279 330727 330785 331197 331290 331693 331840 332127 332317 332666 332934 333296 333487 333757 333838 334215 334485 334745 334924 335131 335568 335627 336012 336252 336568 336769 337067 337278 337499 337676 338010 338383 338606 338771 338957 339301 339476 339949 340137 340465 340482 340856 341145 341249 341649 341891

$ mkdir /KazMuzik/uscis.gov
$ cd /KazMuzik/uscis.gov
$ tar zxvpf .../nutch-0.9.tar.gz
$ cd nutch-0.9
$ vi conf/nutch-site.xml
$ mkdir seeds
$ vi seeds/seeds.txt
$ cat seeds/seeds.txt
http://www.uscis.gov/portal/site/uscis
$ vi conf/crawl-urlfilter.txt
$ cat conf/crawl-urlfilter.txt
-^(file|ftp|mailto):
-\.(gif|GIF|jpg|JPG|png|PNG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|jpeg|JPEG|bmp|BMP)$
# -[?*!@=]
-.*(/.+?)/.*?\1/.*?\1/
+^http://([a-z0-9]*\.)*uscis.gov/
-.
$ export NUTCH_JAVA_HOME=/usr/java/jdk
$ nohup bin/nutch crawl seeds &
$ tail -f nohup.out
crawl started in: crawl-20080705105148
rootUrlDir = seeds
threads = 10
depth = 5
Injector: starting
...
merging indexes to: crawl-20080705105148/index
Adding crawl-20080705105148/indexes/part-00000
done merging
crawl finished: crawl-20080705105148
^C
$ bin/nutch readseg -list -dir crawl-20080705105148/segments | sort
20080705105152  1               2008-07-05T10:51:56     2008-07-05T10:51:56     1       1
20080705105202  33              2008-07-05T10:52:06     2008-07-05T10:52:42     39      24
20080705105254  248             2008-07-05T10:52:57     2008-07-05T10:58:28     287     184
20080705105910  1146            2008-07-05T10:59:22     2008-07-05T11:29:36     1191    903
20080705113222  2998            2008-07-05T11:32:29     2008-07-05T12:53:59     3046    2196
NAME            GENERATED       FETCHER START           FETCHER END             FETCHED PARSED
$ 

$ cp nutch-0.9.war /usr/loca/tomcat/webapps/uscis.war
$ /usr/local/tomcat/bin/shutdown.sh
$ vi /usr/local/tomcat/webapps/uscis/WEB-INF/classes/nutch-site.xml
$ cat /usr/local/tomcat/webapps/uscis/WEB-INF/classes/nutch-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>searcher.dir</name>
    <value>/KazMuzik/uscis.gov/nutch-0.9/crawl-20080705105148</value>
  </property>
</configuration>
$ /usr/local/tomcat/bin/startup.sh
$ 

package net.java.sampo.immigration.processingTime.db;

import javax.sql.DataSource;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CreateTableCommand implements Command {
  private static Log log = LogFactory.getLog(CreateTableCommand.class);

  private static final String sql
    = "create table PROCESSINGTIME ("
    + "FACILITY varchar(32),"
    + "POSTEDDATE timestamp,"
    + "LINENO integer,"
    + "FORM varchar(8),"
    + "TITLE varchar(128),"
    + "CLASSIFICATION varchar(128),"
    + "TIMEFRAME varchar(32),"
    + "PROCESSINGDATE timestamp,"
    + "DAYS integer,"
    + "primary key(FACILITY, POSTEDDATE, LINENO))";

  private JdbcTemplate jdbcTemplate;

  public CreateTableCommand() {
  }

  public void setDataSource(DataSource dataSource) {
    jdbcTemplate = new JdbcTemplate(dataSource);
  }

  public void execute() {
    execute(new ContextBase());
  }

  public boolean execute(Context context) {
    try {
      jdbcTemplate.execute(sql);
    }
    catch (DataAccessException e) {
      log.error(e);
      return true;
    }
    return false;
  }

  public static void main(String[] args) throws Exception {
    Resource resource = new ClassPathResource("processingTime.xml");
    XmlBeanFactory factory = new XmlBeanFactory(resource);
    log.info("getting command bean ..");
    CreateTableCommand command
      = (CreateTableCommand)factory.getBean("createTableCommand");
    log.info("executing ...");
    command.execute();
  }
} 

<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="
      http://www.springframework.org/schema/beans
      http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
      http://www.springframework.org/schema/context
      http://www.springframework.org/schema/context/spring-context-2.5.xsd
    ">

  <bean id="createTableCommand"
     class="net.java.sampo.immigration.processingTime.db.CreateTableCommand">
    <property name="dataSource" ref="dataSource" />
  </bean>

  <bean id="dataSource" destroy-method="close"
     class="org.apache.commons.dbcp.BasicDataSource">
    <property name="driverClassName"
             value="org.apache.derby.jdbc.EmbeddedDriver" />
    <property name="url"
             value="jdbc:derby:processingTime.db;create=true" />
  </bean>
</beans>

package net.java.sampo.immigration.processingTime;

import java.io.Serializable;
import java.util.Date;
import java.util.Formatter;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static net.java.sampo.immigration.processingTime.ParserUtils.parseDate;

public class ProcessingTime implements Serializable {
  private static final Log log = LogFactory.getLog(ProcessingTime.class);

  private String facility;
  private Date date;
  private int lineNo;
  private String form;
  private String title; // name
  private String classification; // basis
  private String timeframe;

  public ProcessingTime(String facility, Date date, int lineNo,
           String form, String name, String timeframe) {
    this(facility, date, lineNo, form, name, null, timeframe);
  }

  public ProcessingTime(String facility, Date date, int lineNo,
           String form, String title, String classification, String timeframe) {
    this.facility = facility;
    this.date = date;
    this.lineNo = lineNo;
    this.form = form;
    this.title = title;
    this.classification = classification;
    this.timeframe = timeframe;
  }

  public String getFacility() {
    return facility;
  }

  public Date getDate() {
    return date;
  }

  public int getLineNO() {
    return lineNo;
  }

  public String getForm() {
    return form;
  }

  public String getTitle() {
    return title;
  }

  public String getName() {
    return title;
  }

  public String getClassification() {
    return classification;
  }

  public String getTimeframe() {
    return timeframe;
  }

  public String toString() {
    return toString(false);
  }

  public String toString(boolean additionalInfo) {
    Formatter f = new Formatter();
    f.format("[%s|%tF|%d|%s|%s|", facility, date, lineNo, form, title);
    if (classification != null) {
      f.format("%s|", classification);
    }
    f.format("%s", timeframe);
    if (additionalInfo) {
      f.format("|%tF|%d", getProcessingDate(), getTimeframeInDay());
    }
    f.format("]");
    return f.toString();
  }

  public Date getProcessingDate() {
    Date postedDate = date;
    if (timeframe == null || timeframe.length() == 0) {
      return null;
    }
    String s = timeframe.toLowerCase();
    char c = s.charAt(0);
    if (c >= 'a' && c <= 'z') {
      return parseDate(s);
    }
    int i = s.indexOf(' ');
    int n = 0 - Integer.parseInt(s.substring(0,i));
    s = s.substring(i+1);
    if (s.startsWith("day")) {
      return DateUtils.addDays(postedDate, n);
    }
    else if (s.startsWith("week")) {
      return DateUtils.addWeeks(postedDate, n);
    }
    else if (s.startsWith("month")) {
      return DateUtils.addMonths(postedDate, n);
    }
    else if (s.startsWith("year")) {
      return DateUtils.addYears(postedDate, n);
    }
    else if (s.startsWith("semana")) {
      return DateUtils.addWeeks(postedDate, n);
    }
    else if (s.startsWith("mese")) {
      return DateUtils.addMonths(postedDate, n);
    }
    else {
      log.warn("invalid time frame \"" + timeframe + "\"");
      return null;
    }
  }

  public int getTimeframeInDay() {
    Date postedDate = date;
    Date processingDate = getProcessingDate();
    if (processingDate == null) {
      return -1;
    }
    return (int)((postedDate.getTime() - processingDate.getTime()) / 86400000L);
    // 86400000 = 24 x 60 x 60 x 1000
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class ParserUtils {
  private static Log log = LogFactory.getLog(ParserUtils.class);

  public static String toTrimedString(Reader in) throws IOException {
    StringBuilder sb = new StringBuilder();
    for (String line : (List<String>)IOUtils.readLines(in)) {
      sb.append(line.trim());
    }
    return sb.toString();
  }

  public static String cutContent(byte[] bytes) throws IOException {
    Reader in = new InputStreamReader(new ByteArrayInputStream(bytes), "US-ASCII");
    return cutContent( toTrimedString(in) );
  }

  public static String cutContent(File file) throws IOException {
    Reader in = new InputStreamReader(new FileInputStream(file), "US-ASCII");
    return cutContent(in);
  }

  public static String cutContent(Reader in) throws IOException {
    return cutContent( toTrimedString(in) );
  }

  public static String cutContent(String html) {
    String sl = html.toLowerCase();
    int n = sl.indexOf("posted ");
    n = sl.indexOf("posted ", n+7);

    int m1 = sl.lastIndexOf("<b><big>", n);
    if (m1 < 0) { // NBC 2003-11-14
      m1 = n;
    }
    m1 = sl.lastIndexOf(">", m1);
    int m2 = sl.indexOf("<", n);
    StringBuilder sb = new StringBuilder();
    sb.append("<p align=\"center\">");
    sb.append(html.substring(m1+1,m2));
    sb.append("</p>");

    m1 = sl.indexOf("<table", m2);
    m2 = sl.indexOf("</table>", m1);
    sb.append(html.substring(m1,m2+8));
    return sb.toString();
  }

  public static String parseFacilityName(String content) {
    String s = content.toLowerCase();
    int n = s.indexOf("<big>");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"<big> \" tag not found.");
      }
      return "National Benefits Center"; // NBC 2003-11-14
    }
    int m = s.indexOf("</", n+5); // 2003-11-14
    if (m < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"</big> \" tag not found.");
      }
      return null;
    }
    String name = content.substring(n+5, m).trim();
    // 2003-11-14
    if (s.substring(0,n).indexOf("service center") >= 0
     && name.toLowerCase().indexOf("service center") < 0) {
      name = name + " Service Center";
    }
    return name;
  }

  public static Date parsePostedDate(String content) {
    int n = content.toLowerCase().indexOf("posted ");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"Posted \" not found.");
      }
      return null;
    }
    return parseDate(content.substring(n+7));
  }

  public static ProcessingTime[] parseProcessingTimes(String content) {
    String facility = parseFacilityName(content);
    Date date = parsePostedDate(content);
    int lineNo = 0;
    //
    String s = content.toLowerCase();
    int n = s.indexOf("form");
    List<ProcessingTime> list = new ArrayList<ProcessingTime>();
    while (true) {
      n = s.indexOf("<tr", n);
      if (n < 0) {
        break;
      }
      int m = s.indexOf("<tr", n+4); // CSC 2003-11-14 </tr> missing !!
      String line = null;
      if (m < 0) {
        line = content.substring(n);
      }
      else {
        line = content.substring(n, m);
      }
      list.add( parseLine(facility, date, lineNo, line) );
      lineNo ++;
      if (m < 0) {
        break;
      }
      n = m;
    }
    ProcessingTime[] array = new ProcessingTime[ list.size() ];
    list.toArray(array);
    return array;
  }

  private static ProcessingTime parseLine(String facility, Date date, int lineNo, String s) {
    if (log.isDebugEnabled()) {
      log.debug(s);
    }
    String sl = s.toLowerCase();
    int m = sl.indexOf("<b>");
    int n = sl.indexOf("<", m+3); // 2003-11-14
    List<String> list = new ArrayList<String>(4);
    list.add(s.substring(m+3, n).trim()); // form
    if (log.isDebugEnabled()) {
      log.debug(list);
    }
    while (true) {
      n = sl.indexOf("</td", n+5); // 2005-10-19 District Office
      if (n < 0) {
        break;
      }
      m = sl.lastIndexOf(">", n);
      list.add( s.substring(m+1, n).trim() );
      if (log.isDebugEnabled()) {
        log.debug(list);
      }
    }
    ProcessingTime processingTime = null;
    if (list.size() == 3) {
      processingTime = new ProcessingTime(facility, date, lineNo,
                             list.get(0), list.get(1), list.get(2));
    }
    else if (list.size() >= 4) {
      processingTime = new ProcessingTime(facility, date, lineNo,
                             list.get(0), list.get(1), list.get(2), list.get(3));
    }
    else {
      if (log.isWarnEnabled()) {
        log.warn("invalid field number = " + list.size() + " for " + s);
      }
    }
    return processingTime;
  }

  public static Date parseDate(String s) {
    int n = s.indexOf(' ');
    int m = s.indexOf(',', n+1);
    int dd = Integer.parseInt(s.substring(n+1,m));
    int yyyy = Integer.parseInt(s.substring(m+2,m+6));
    int mm = parseMonth(s.substring(0,n));
    if (mm < 1 || mm > 12) {
      return null;
    }
    Calendar cal = Calendar.getInstance();
    cal.set(yyyy, mm-1, dd, 12, 0, 0);
    return cal.getTime();
  }

  private static final String[] months = {
    "jan", "feb", "mar", "apr", "may", "jun",
    "jul", "aug", "sep", "oct", "nov", "dec"
  };

  private static int parseMonth(String s) {
    s = s.toLowerCase();
    for (int i = 0; i < months.length; i++) {
      if (s.startsWith(months[i])) {
        return i+1;
      }
    }
    return -1;
  }
} 

<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="
      http://www.springframework.org/schema/beans
      http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
      http://www.springframework.org/schema/context
      http://www.springframework.org/schema/context/spring-context-2.5.xsd
    ">

  <bean id="chain" class="net.java.sampo.immigration.processingTime.command.spring.CommandChain">
    <property name="commands">
      <list>
        <ref bean="fetchCommand" />
        <ref bean="cutContentCommand" />
        <ref bean="parseContentCommand" />
        <ref bean="setFilenameCommand" />
        <ref bean="writeBytesCommand" />
      </list>
    </property>
    <property name="contextMap">
      <map>
        <entry key="url"
               value="https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70" />
        <entry key="outPath" value="/tmp/pt" />
      </map>
    </property>
  </bean>

  <bean id="fetchCommand"
        class="net.java.sampo.immigration.processingTime.command.FetchCommand" />
  <bean id="cutContentCommand"
        class="net.java.sampo.immigration.processingTime.command.CutContentCommand" />
  <bean id="parseContentCommand"
        class="net.java.sampo.immigration.processingTime.command.ParseContentCommand" />
  <bean id="setFilenameCommand"
        class="net.java.sampo.immigration.processingTime.command.SetFilenameCommand" />
  <bean id="writeBytesCommand"
        class="net.java.sampo.immigration.processingTime.command.WriteBytesCommand" />
</beans>

package net.java.sampo.immigration.processingTime.command.spring;

import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CommandChain extends ChainBase implements InitializingBean {
  private static Log log = LogFactory.getLog(CommandChain.class);

  private List<Command> commands;
  private Map<String,String> contextMap;
  private Context context;

  public CommandChain() {
    context = null;
  }

  public void setCommands(List<Command> commands) {
    this.commands = commands;
  }

  public List<Command> getCommands() {
    return commands;
  }

  public void setContextMap(Map<String,String> contextMap) {
    this.contextMap = contextMap;
  }

  public Map<String,String> getContextMap() {
    return contextMap;
  }

  public Context getContext() {
    return context;
  }

  public void afterPropertiesSet() {
    // adding commands
    for (Command command : commands) {
      addCommand(command);
    }
    // setting context
    context = new ContextBase();
    for (String key : contextMap.keySet()) {
      String value = contextMap.get(key);
      context.put(key, value);
    }
  }

  public void execute() throws Exception {
    super.execute(context);
  }

  public static void main(String[] args) throws Exception {
    Resource resource = new ClassPathResource("chain.xml");
    XmlBeanFactory factory = new XmlBeanFactory(resource);
    log.info("getting chain ..");
    CommandChain chain = (CommandChain)factory.getBean("chain");
    Context context = chain.getContext();
    log.info("context=" + context);
    log.info("executing ...");
    chain.execute();
    //
    context.remove("bytes");
    context.remove("content");
    context.remove("processingTimes");
    log.info("context=" + context);
  }
} 

$ cat log4j.properties
log4j.rootCategory=WARN,A1
log4j.category.net.java.sampo=INFO
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c{1} - %m%n
$ java -classpath .:lib/commons-logging-1.1.1.jar:lib/log4j-1.2.15.jar:lib/commons-chain-1.1.jar\
:lib/commons-io-1.3.2.jar:lib/spring-2.5.jar \
net.java.sampo.immigration.processingTime.command.spring.CommandChain
2007-12-16 07:39:34,508 [main] INFO  CommandChain - getting chain ..
2007-12-16 07:39:34,569 [main] INFO  CommandChain \
- context={outPath=/tmp/pt, url=https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70}
2007-12-16 07:39:34,569 [main] INFO  CommandChain - executing ...
2007-12-16 07:39:36,012 [main] INFO  CommandChain \
- context={outPath=/tmp/pt, facility=San Jose CA, posted=Wed Nov 14 12:00:00 PST 2007,\
 url=https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70,\
 outFile=/tmp/pt/San_Jose_CA-20071114.html}
$ 

package net.java.sampo.immigration.processingTime;

import java.io.InputStream;
import java.io.IOException;
import java.net.URL;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class EgovFetcher {
  private static final long minimumFetchInterval = 1000; // 1 sec

  private static Log log = LogFactory.getLog(EgovFetcher.class);

  private static EgovFetcher fetcher = null;

  public static EgovFetcher getInstance() {
    if (fetcher == null) {
      fetcher = new EgovFetcher();
    }
    return fetcher;
  }

  private long lastFetchedTime;

  public EgovFetcher() {
    lastFetchedTime = -1L;
  }

  synchronized public byte[] fetch(String url) throws IOException {
    if (url == null) {
      log.error("url is null.");
      return null;
    }
    long now = System.currentTimeMillis();
    long t = minimumFetchInterval - (now - lastFetchedTime);
    if (t > 0L) {
      if (log.isInfoEnabled()) {
        log.info("sleeping " + t + "msecs ..");
      }
      try {
        Thread.sleep(t);
      }
      catch (InterruptedException e) {
      }
      now = System.currentTimeMillis();
    }
    lastFetchedTime = now;
    byte[] bytes = null;
    InputStream in = new URL(url).openStream();
    bytes = IOUtils.toByteArray(in);
    return bytes;
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class FacilityUrlMap extends HashMap<String, String> {
  private static final String ptimesUrl = " https://egov.uscis.gov/cris/jsps/ptimes.jsp";
  private static final String officeUrl
    = "https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=";
  private static final String centerUrl
    = "https://egov.uscis.gov/cris/jsps/Processtimes.jsp?SeviceCenter=";
  private static final String nbcUrl = "https://egov.uscis.gov/cris/jsps/NBCprocesstimes.jsp";

  private static Log log = LogFactory.getLog(FacilityUrlMap.class);

  private static FacilityUrlMap map;

  public static FacilityUrlMap getInstance() {
    if (map == null) {
      map = new FacilityUrlMap();
    }
    return map;
  }

  private String[] facilityNames;

  public FacilityUrlMap() {
    super();
    try {
      init();
    }
    catch (IOException e) {
      log.fatal(e);
    }
  }

  public String getUrl(String facility) {
    return get(facility);
  }

  public String[] getAllFacilityNames() {
    return facilityNames;
  }

  private void init() throws IOException {
    byte[] bytes = EgovFetcher.getInstance().fetch(ptimesUrl);   
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new ByteArrayInputStream(bytes), "ISO-8859-1"));
    parse(in);
  }

  private void parse(BufferedReader in) throws IOException {
      List<String> list = new ArrayList<String>();
      int mode = 0;
      while (true) {
        String line = in.readLine();
        if (log.isDebugEnabled()) {
          log.debug("" + mode + " : " + line);
        }
        if (line == null) {
          break;
        }
        String s = line.toLowerCase();
        if (mode == 0) {
          if (s.indexOf("<select name=\"selectedoffice\">") >= 0) {
            mode = 1;
          }
          else if (s.indexOf("<select name=\"sevicecenter\">") >= 0) { // Sevice : "r" missing !!
            mode = 2;
          }
          continue;
        }
        if (s.indexOf("</select>") >= 0) {
          if (mode == 2) {
            break;
          }
          mode = 0;
          continue;
        }
        //
        int n = 0;
        while (true) {
          int m = s.indexOf("<option value=\"", n);
          if (m < 0) {
            break;
          }
          n = s.indexOf('"', m+15);
          String id = line.substring(m+15,n).trim();
          m = s.indexOf('<', n+2);
          String name = line.substring(n+2, m).trim();
          log.info(id + " : " + name);
          if (mode == 1) {
            int intId = Integer.parseInt(id);
            put(name, officeUrl + intId);
            list.add(name);
          }
          else if (mode == 2) {
            put(name, centerUrl + id);
            list.add(name);
          }
          n = m+8;
        }
      }
      String name = "National Benefits Center";
      put(name, nbcUrl);
      list.add(name);
      //
      int n = list.size();
      facilityNames = new String[n];
      list.toArray(facilityNames);
  }
} 

package net.java.sampo.immigration.processingTime.command;

import java.io.File;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.chain.Chain;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import net.java.sampo.immigration.processingTime.FacilityUrlMap;

public class FetchAllCommand implements Command {
  private static Log log = LogFactory.getLog(FetchAllCommand.class);

  public boolean execute(Context context) {
    FacilityUrlMap urlMap = FacilityUrlMap.getInstance();
    if (urlMap == null) {
      log.fatal("facility to url map is null.");
      return true;
    }
    String outPath = (String)context.get("outPath");
    if (outPath == null) {
      log.error("outPath not set in context.");
      return true;
    }
    File outDir = new File(outPath);
    if (! outDir.exists()) {
      log.warn("outPath not exist. creating..");
      if (! outDir.mkdir()) {
        log.warn("outPath failed to create.");
        return true;
      }
    }
    Chain command = new ChainBase();
    command.addCommand(new FetchCommand());
    command.addCommand(new CutContentCommand());
    command.addCommand(new ParseContentCommand());
    command.addCommand(new SetFilenameCommand());
    command.addCommand(new WriteBytesCommand());

    for (String facility : urlMap.getAllFacilityNames()) {
      String url = urlMap.getUrl(facility);
      Context eachContext = new ContextBase();
      eachContext.put("url", url);
      eachContext.put("outPath", outPath);
      if (log.isInfoEnabled()) {
        log.info(facility + " : " + url + " to " + outPath);
      }
      try {
        command.execute(eachContext);
      }
      catch (Exception e) {
        log.error(e);
      }
    }
    return false;
  }

  public static void main(String[] args) throws Exception {
    String path = "orig";
    if (args.length > 0) {
      path = args[0];
    }
    Context context = new ContextBase();
    context.put("outPath", path);
    Chain command = new ChainBase();
    command.addCommand(new FetchAllCommand());
    command.execute(context);
  }
} 

The Case Status Online, Processing Times, Office Locator, and Change of Address Online Systems \
will be experiencing a scheduled outage from 9:30 PM EST
on Friday, December 7 to 8:00 AM EST on Monday, December 10 due to scheduled maintenance. 

package net.java.sampo.immigration.processingTime;

import java.io.Serializable;

public class ProcessingTime implements Serializable {
  private String form;
  private String title; // name
  private String classification; // basis
  private String timeframe;

  public ProcessingTime(String form, String name, String timeframe) {
    this(form, name, null, timeframe);
  }

  public ProcessingTime(String form, String title, String classification, String timeframe) {
    this.form = form;
    this.title = title;
    this.classification = classification;
    this.timeframe = timeframe;
  }

  public String getForm() {
    return form;
  }

  public String getTitle() {
    return title;
  }

  public String getName() {
    return title;
  }

  public String getClassification() {
    return title;
  }

  public String getTimeframe() {
    return timeframe;
  }

  public String toString() {
    return "PT[" + form + "|" + title + "|"
         + ((classification==null)?"":(classification+"|"))
         + timeframe + "]";
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static net.java.sampo.immigration.processingTime.ProcessingTimeUtils.parseDate;

public class ProcessingTimesPageParser extends BufferedReader {
  private static Log log = LogFactory.getLog(ProcessingTimesPageParser.class);

  private String facilityName;
  private Date date;
  private ProcessingTime[] processingTimes;
  private String content;

  public ProcessingTimesPageParser(Reader in) {
    super(in);
  }

  public ProcessingTime[] getProcessingTimes() {
    return processingTimes;
  }

  public String getFacilityName() {
    return facilityName;
  }

  public Date getPostedDate() {
    return date;
  }

  public String getContent() {
    return content;
  }

  public void parseAll() throws IOException {
    content = parseContent();
    if (log.isDebugEnabled()) {
      log.debug(content);
    }
    facilityName = parseFacility(content);
    date = parsePostedDate(content);
    processingTimes = parseTable(content);
  }

  private String parseContent() throws IOException {
    int postedCount = 0;
    int endTableCount = 0;
    StringBuilder sb = new StringBuilder();
    String tableTag = null;
    while (true) {
      String line = readLine();
      if (line == null) {
        break;
      }
      line = line.trim();
      String lineLowerCase = line.toLowerCase();
      if (lineLowerCase.indexOf("<table") >= 0 && postedCount < 2) {
        tableTag = line;
      }
      if (lineLowerCase.indexOf("<tr") >= 0 && postedCount < 2) {
        sb = new StringBuilder();
        sb.append(tableTag);
        if (line.indexOf("<tbody") < 0) {
          sb.append("<tbody>");
        }
      }
      sb.append(line);
      if (lineLowerCase.indexOf("posted ") >= 0) {
        postedCount ++;
      }
      if (lineLowerCase.indexOf("</table>") >= 0 && postedCount >= 2) {
        endTableCount ++;
        if (endTableCount >= 2) {
          break;
        }
      }
    }
    return sb.toString();
  }

  private String parseFacility(String content) {
    String contentLowerCase = content.toLowerCase();
    int n = contentLowerCase.indexOf("<big>");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"<big> \" tag not found.");
      }
      return null;
    }
    int m = contentLowerCase.indexOf("</big>", n+5);
    if (m < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"</big> \" tag not found.");
      }
      return null;
    }
    return content.substring(n+5, m);
  }

  private Date parsePostedDate(String content) {
    int n = content.toLowerCase().indexOf("posted ");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"Posted \" not found.");
      }
      return null;
    }
    return parseDate(content.substring(n+7));
  }

  private ProcessingTime[] parseTable(String content) {
    String contentLowerCase = content.toLowerCase();
    int n = contentLowerCase.indexOf("processing timeframe");
    List<ProcessingTime> list = new ArrayList<ProcessingTime>();
    while (true) {
      n = contentLowerCase.indexOf("<tr", n);
      if (n < 0) {
        break;
      }
      int m = contentLowerCase.indexOf("</tr>", n+4);
      list.add( parseLine( content.substring(n, m+5) ) );
      n = m;
    }
    ProcessingTime[] array = new ProcessingTime[ list.size() ];
    list.toArray(array);
    return array;
  }

  private ProcessingTime parseLine(String s) {
    if (log.isDebugEnabled()) {
      log.debug(s);
    }
    String sl = s.toLowerCase();
    int m = sl.indexOf("<b>");
    int n = sl.indexOf("</b>", m+3);
    List<String> list = new ArrayList<String>(4);
    list.add(s.substring(m+3, n).trim()); // form
    if (log.isDebugEnabled()) {
      log.debug(list);
    }
    while (true) {
      n = sl.indexOf("</td>", n+5);
      if (n < 0) {
        break;
      }
      m = sl.lastIndexOf(">", n);
      list.add( s.substring(m+1, n).trim() );
      if (log.isDebugEnabled()) {
        log.debug(list);
      }
    }
    ProcessingTime processingTime = null;
    if (list.size() == 3) {
      processingTime = new ProcessingTime(list.get(0), list.get(1), list.get(2));
    }
    else if (list.size() == 4) {
      processingTime = new ProcessingTime(list.get(0), list.get(1), list.get(2), list.get(3));
    }
    else {
      if (log.isWarnEnabled()) {
        log.warn("invalid field number = " + list.size());
      }
    }
    return processingTime;
  }

  public static void main(String[] args) throws Exception {
    ProcessingTimesPageParser parser
      = new ProcessingTimesPageParser(new InputStreamReader(System.in, "ISO-8859-1"));
    parser.parseAll();
    System.out.printf("%s : %tF%n", parser.getFacilityName(), parser.getPostedDate());
    for (ProcessingTime pt : parser.getProcessingTimes()) {
      System.out.println(pt.toString());
    }
    System.out.println(parser.getContent());
  }
} 

package net.java.sampo.immigration.processingTime;

import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class ProcessingTimeUtils {
  private static Log log = LogFactory.getLog(ProcessingTimeUtils.class);

  public static Date parseDate(String s) {
    int n = s.indexOf(' ');
    int m = s.indexOf(',', n+1);
    int dd = Integer.parseInt(s.substring(n+1,m));
    int yyyy = Integer.parseInt(s.substring(m+2,m+6));
    int mm = parseMonth(s.substring(0,n));
    if (mm < 1 || mm > 12) {
      return null;
    }
    Calendar cal = Calendar.getInstance();
    cal.set(yyyy, mm-1, dd);
    return cal.getTime();
  }

  private static final String[] months = {
    "jan", "feb", "mar", "apr", "may", "jun",
    "jul", "aug", "sep", "oct", "nov", "dec"
  };

  private static int parseMonth(String s) {
    s = s.toLowerCase();
    for (int i = 0; i < months.length; i++) {
      if (s.startsWith(months[i])) {
        return i+1;
      }
    }
    return -1;
  }
}