$ curl http://kazuomik.livejournal.com/[2006-2009]/[01-12]/ -o "#1-#2.html"
$ ls
2006-01.html	2006-11.html	2007-09.html	2008-07.html	2009-05.html
2006-02.html	2006-12.html	2007-10.html	2008-08.html	2009-06.html
2006-03.html	2007-01.html	2007-11.html	2008-09.html	2009-07.html
2006-04.html	2007-02.html	2007-12.html	2008-10.html	2009-08.html
2006-05.html	2007-03.html	2008-01.html	2008-11.html	2009-09.html
2006-06.html	2007-04.html	2008-02.html	2008-12.html	2009-10.html
2006-07.html	2007-05.html	2008-03.html	2009-01.html	2009-11.html
2006-08.html	2007-06.html	2008-04.html	2009-02.html	2009-12.html
2006-09.html	2007-07.html	2008-05.html	2009-03.html
2006-10.html	2007-08.html	2008-06.html	2009-04.html
$ 

$ cat *.html | grep 'm:  <a href="http://kazuomik.livejournal.com/' \
| sed -e 's/^.*livejournal\.com\///' -e 's/\.html\">.*$//' \
| sort -k1nr \
> id.txt
$ cat id.txt
341891
341649
341249
341145
340856
340482
340465
340137
339949
339476
...
2603
2409
2080
1960
1598
1332
1120
1008
722
495
$ wc -l id.txt
    1335 id.txt
$ 

$ cat curl-all.sh
#!/bin/sh
for N in `cat id.txt`
do
  curl http://kazuomik.livejournal.com/${N}.html -o ${N}.html
done
$ sh curl-all.sh
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 55323  100 55323    0     0   124k      0 --:--:-- --:--:-- --:--:--  630k

...
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 52824  100 52824    0     0  78390      0 --:--:-- --:--:-- --:--:--  142k
$ 

KazMuzik.net

Music / Technology / Immigration / アメリカ

145303 <- 205797 205872 206190 206548 206825 207062 207320 207493 207679 207967 208347 208527 208712 209045 209208 209509 209845 210114 210264 210456 210825 211054 211268 211494 211799 212195 212373 212731 212843 213073 213444 213707 213801 214104 214389 214736 214893 215047 215315 215731 215896 216117 216402 216649 216851 217187 217415 217728 217981 218151 218420 218709 219104 219369 219446 219810 220035 220198 220575 220737 221027 221433 221541 221736 221961 222220 222634 222901 223216 223430 223586 223981 224060 224306 224532 224991 225133 225388 225644 225982 226153 226485 226617 226993 227201 227374 227695 227993 228312 228505 228744 229048 229180 229579 229743 230024 230164 230637 230728 230963 231309 231574 231695 232129 232338 232677 232828 232990 233467 233632 233937 234034 234368 234695 234878 235220 235443 235570 235811 236189 236454 236592 236948 237064 237459 237724 237905 238245 238339 238780 238890 239183 239368 239642 239888 240213 240581 240649 240923 241334 241577 241773 241997 242330 242620 242890 243158 243211 243506 243718 243969 244328 244599 244918 245219 245345 245506 245909 246163 246503 246649 246911 247204 247344 247683 247889 248127 248344 248751 248898 249161 249526 249674 249934 250271 250529 250906 251272 250653 251489 251840 251951 252405 252671 252771 253029 253243 253629 253820 254021 254251 254480 254946 255198 255304 255640 255949 256078 256502 256703 256845 257525 257544 257900 257251 258440 258634 258182 258941 259137 259519 259828 259950 260212 260494 260839 261095 261166 261588 261883 262066 262188 262614 262783 262921 263237 263622 263725 264181 264363 264658 264712 265221 265167 265536 265925 266110 266295 266549 266967 267127 267328 267647 267861 268736 268895 269191 268120 269357 268352 269692 269950 270110 270403 270783 271091 271270 271586 271847 272056 272253 272468 272697 272960 273285 273630 273870 273962 274185 274647 274892 275049 275352 275678 275836 276048 276338 276735 276894 277104 277413 277535 277946 278133 278287 278772 278925 279261 279364 279561 280046 280185 280341 280590 280979 281294 281502 281829 281953 282338 282622 282655 283151 283102 283565 283776 284137 284220 284727 285077 284626 285191 285656 285936 286066 286281 286566 286959 287091 287327 287491 287761 288043 288396 288734 288966 289104 289456 289641 290029 290293 290438 290775 291029 291096 291348 291706 291958 292148 292606 292672 293035 293173 293581 293678 293995 294277 294433 294904 294944 295198 295531 295688 295997 296414 296580 296931 297056 297332 297661 297899 297991 298462 298686 298777 299075 299453 299586 300028 300113 300486 300757 301029 301444 301676 301952 302286 302621 302501 301184 302896 303252 303589 303792 304046 304149 304544 304804 305572 305114 305269 305778 306118 306402 306496 306769 306984 307302 307688 307964 308195 308344 308546 308841 309232 309314 309711 309854 310041 310417 310567 310959 311092 311382 311576 311970 312300 312491 312752 313010 313181 313538 313735 313969 314318 314376 314692 315097 315235 315430 315743 315996 316411 316610 316803 316951 317267 317489 317737 318044 318447 318562 318767 319216 319292 319520 319908 320021 320329 320709 320902 321236 321392 321568 322004 322079 322341 322682 322961 323152 323471 323783 323956 324204 324582 324828 324945 325267 325563 325791 325930 326237 326655 326805 327140 327360 327669 327925 328132 328223 328489 328819 329120 329442 329486 329825 330126 330279 330727 330785 331197 331290 331693 331840 332127 332317 332666 332934 333296 333487 333757 333838 334215 334485 334745 334924 335131 335568 335627 336012 336252 336568 336769 337067 337278 337499 337676 338010 338383 338606 338771 338957 339301 339476 339949 340137 340465 340482 340856 341145 341249 341649 341891

#!/bin/sh

name="kazmuzik"
date=`date +"%Y%m%d"`
tmpfile="tmp-${date}.xml"
xmlfile="${name}-${date}.xml"
url="http://twitter.com/statuses/user_timeline.xml"

curl "${url}?screen_name=${name}&count=200&page=1" > $tmpfile
n=`cat $tmpfile | wc -l`
n=`expr $n - 1`
head -$n $tmpfile > $xmlfile
rm -f $tmpfile

curl "${url}?screen_name=${name}&count=200&page=2" | tail +3 >> $xmlfile

package net.kazmuzik.twitter;

import java.io.Serializable;
import java.util.Date;

public class TwitterStatus implements Serializable {
  private Date time;
  private long id;
  private String text;

  public TwitterStatus() {
  }

  public Date getTime() {
    return time;
  }

  public long getId() {
    return id;
  }

  public String getText() {
    return text;
  }

  public void setTime(Date time) {
    this.time = time;
  }

  public void setId(long id) {
    this.id = id;
  }

  public void setText(String text) {
    this.text = text;
  }
} 

package net.kazmuzik.twitter;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Reader;
import java.util.Date;

public class TwitterXMLReader extends BufferedReader {
  private static final boolean debug = false;

  private static final String tag0 = "<created_at>";
  private static final String tag1 = "<id>";
  private static final String tag2 = "<text>";
  private static final String tag3 = "</status>";
  private static final int tag0len = tag0.length();
  private static final int tag1len = tag1.length();
  private static final int tag2len = tag2.length();

  public TwitterXMLReader(Reader in) throws IOException {
    super(in);
  }

  public TwitterXMLReader(InputStream in) throws IOException {
    super(new InputStreamReader(in, "UTF-8"));
  }

  public TwitterStatus readStatus() throws IOException {
    TwitterStatus status = new TwitterStatus();
    int tag = 0;
    while (true) {
      String line = readLine();
      if (line == null) {
        return null;
      }
      if (debug) {
        System.err.printf("%d: %s%n", tag, line.trim());
      }
      if (tag == 0) {
        int n = line.indexOf(tag0);
        if (n < 0) {
          continue;
        }
        n += tag0len;
        int m = line.indexOf('<', n);
        if (m < 0) {
          return null;
        }
        Date time = parseTime(line.substring(n,m));
        status.setTime(time);
        if (debug) {
          System.err.printf("#: time=%TF %TT%n", time, time);
        }
        tag = 1;
        continue;
      }
      else if (tag == 1) {
        int n = line.indexOf(tag1);
        if (n < 0) {
          continue;
        }
        n += tag1len;
        int m = line.indexOf('<', n);
        if (m < 0) {
          return null;
        }
        long id = Long.parseLong(line.substring(n,m));
        status.setId(id);
        if (debug) {
          System.err.printf("#: id=%d%n", id);
        }
        tag = 2;
        continue;
      }
      else if (tag == 2) {
        int n = line.indexOf(tag2);
        if (n < 0) {
          continue;
        }
        n += tag2len;
        int m = line.indexOf('<', n);
        if (m < 0) {
          return null;
        }
        String text = convertEntities(line.substring(n,m));
        status.setText(text);
        if (debug) {
          System.err.printf("#: text=%s%n", text);
        }
        tag = 3;
        continue;
      }
      else if (tag == 3) {
        int n = line.indexOf(tag3);
        if (n < 0) {
          continue;
        }
        return status;
      }
    }
  }

  private static String convertEntities(String text) {
    StringBuilder sb = new StringBuilder();
    int n = 0;
    int x = 0;
    for (char c : text.toCharArray()) {
      if (n == 0) {
        if (c == '&') {
          n = 1;
        }
        else {
          sb.append(c);
        }
        continue;
      }
      else if (n == 1) {
        if (c == '#') {
          n = 2;
          x = 0;
        }
        else {
          sb.append('&');
          sb.append(c);
          n = 0;
        }
        continue;
      }
      else if (n == 2) {
        if (c == ';') {
          sb.append((char)x);
          n = 0;
        }
        else if (c < '0' || c > '9') {
          sb.append(c);
          n = 0;
        }
        else {
          x *= 10;
          x += (c - '0');
        }
        continue;
      }
    }
    return sb.toString();
  }

  private static Date parseTime(String s) {
    return new Date(s);
  }
} 

import java.io.InputStream;
import java.io.FileInputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.util.Date;
import net.kazmuzik.twitter.TwitterStatus;
import net.kazmuzik.twitter.TwitterXMLReader;

public class TwitterXMLConverter {
  public static void main(String[] args) throws Exception {
    InputStream is = System.in;
    if (args.length > 0) {
      is = new FileInputStream(args[0]);
    }
    TwitterXMLReader in = new TwitterXMLReader(is);
    PrintWriter out = new PrintWriter(new OutputStreamWriter(System.out, "UTF-8"));
    out.println("<html><head>");
    out.println("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" />");
    out.println("</head><body>");
    out.println("<ul>");
    Date prevTime = null;
    while (true) {
      TwitterStatus status = in.readStatus();
      if (status == null) {
        break;
      }
      Date time = status.getTime();
      String text = addATag(status.getText());
      if (prevTime != null) {
        if (time.getDay() != prevTime.getDay()) {
          out.println("</ul>");
          out.println("<ul>");
        }
      }
      out.printf("<li><i><font color=#0000cc>%TF %TT</font> "
               + "<a href=\"http://twitter.com/kazmuzik\">"
               + "<font color=#00cc00>#%d</font></a></i><br>",
                 time, time, status.getId());
      out.printf("%s<br>&nbsp;</li>%n", text);
      prevTime = time;
    }
    out.println("</ul>");
    out.println("</body></html>");
    out.flush();
    out.close();
  }

  private static String addATag(String text) {
    StringBuilder sb = new StringBuilder();
    int n = 0;
    while (true) {
      int m = text.indexOf("http://", n);
      if (m < 0) {
        sb.append(text.substring(n));
        break;
      }
      sb.append(text.substring(n,m));
      n = text.indexOf(' ', m);
      String url = null;
      if (n < 0) {
        url = text.substring(m);
      }
      else {
        url = text.substring(m, n);
      }
      sb.append("<a href=\"");
      sb.append(url);
      sb.append("\">");
      sb.append(url);
      sb.append("</a>");
      if (n < 0) {
        break;
      }
    }
    return sb.toString();
  }
} 

$ mkdir classes
$ javac -d classes *.java
$ java -classpath classes TwitterXMLConverter kazmuzik.xml > kazmuzik.html
$ 

$ curl 'http://twitter.com/statuses/user_timeline.xml?screen_name=kazmuzik&count=200' > kazmuzik.xml
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  414k  100  414k    0     0   603k      0 --:--:-- --:--:-- --:--:--  809k
$ cat kazmuzik.xml
<?xml version="1.0" encoding="UTF-8"?>
<statuses type="array">
<status>
  <created_at>Sun May 10 15:38:12 +0000 2009</created_at>
  <id>1755243926</id>
  <text>140&#25991;&#23383;&#12391;&#34920;&#29694;&#12391;&#12365;&#12427;&#20869;&#23481;&#12391;&#12377;\
&#12364;&#12289;&#33521;&#35486;&#12384;&#12392;&#26085;&#26412;&#35486;&#12398;&#21322;&#20998;&#20197;\
&#19979;&#12395;&#12394;&#12387;&#12390;&#12375;&#12414;&#12356;&#12414;&#12377;&#12290;&#12354;&#12427;\
&#31243;&#24230;&#12289;&#20013;&#36523;&#12364;&#12354;&#12427;&#22580;&#21512;&#12399;&#12289;&#26085;\
&#26412;&#35486;&#12364;&#22311;&#20498;&#30340;&#12395;&#26377;&#21033;&#12391;&#12377;&#12290;&#12371;\
&#12428;&#12418;&#12289;Unicode&#12364;&#26222;&#21450;&#12375;&#12390;&#12365;&#12383;&#12362;&#12363;\
&#12370;&#12391;&#12377;&#12290;</text>
  <source>web</source>
  <truncated>false</truncated>
  <in_reply_to_status_id></in_reply_to_status_id>
  <in_reply_to_user_id></in_reply_to_user_id>
  <favorited>false</favorited>
  <in_reply_to_screen_name></in_reply_to_screen_name>
  <user>
    <id>26517941</id>
    <name>Kaz Muzik</name>
...
    <statuses_count>204</statuses_count>
    <notifications></notifications>
    <following></following>
  </user>
</status>
</statuses>
$ 

$ curl 'http://twitter.com/statuses/user_timeline.xml?screen_name=kazmuzik&count=200&page=2' >> kazmuzik.xml
$ vi kazmuzik.xml
$ 

$ cat PList.java

import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;

public class PList implements Serializable {
  private Map<String,String> dict;

  public PList() {
    dict = new HashMap<String,String>();
  }

  public void put(String key, String value) {
    dict.put(key, value);
  }

  public String get(String key) {
    return dict.get(key);
  }

  public Map<String,String> getDict() {
    return dict;
  }
}

$ cat PListReader.java

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.PrintWriter;

public class PListReader extends BufferedReader {
  public PListReader(Reader in) throws IOException {
    super(in);
  }

  public PListReader(InputStream is) throws IOException {
    this(new InputStreamReader(is, "UTF-8"));
  }

  public PListReader(String filename) throws IOException {
    this(new FileInputStream(filename));
  }

  public String readKey() throws IOException {
    while (true) {
      String line = readLine();
      if (line == null) {
        return null;
      }
      int n = line.indexOf("<key>");
      if (n < 0) {
        continue;
      }
      int m = line.indexOf("</key>", n+5);
      if (m < 0) {
        continue;
      }
      return line.substring(n+5, m);
    }
  }

  public String readStringValue() throws IOException {
    String line = readLine();
    if (line == null) {
      return null;
    }
    int n = line.indexOf("<string>");
    if (n < 0) {
      return null;
    }
    int m = line.indexOf("</string>", n+8);
    if (m < 0) {
      return null;
    }
    return line.substring(n+8, m);
  }

  public String[] readKeyAndStringValue() throws IOException {
    while (true) {
      String key = readKey();
      if (key == null) {
        break;
      }
      String value = readStringValue();
      if (value == null) {
        continue;
      }
      return new String[] { key, value };
    }
    return null;
  }

  public PList readPList() throws IOException {
    PList plist = new PList();
    while (true) {
      String[] pair = readKeyAndStringValue();
      if (pair == null) {
        break;
      }
      String key = pair[0];
      String value = pair[1];
      plist.put(key, value);
    }
    return plist;
  }

  public void printPList(PrintWriter out, String separator) throws IOException {
    while (true) {
      String[] pair = readKeyAndStringValue();
      if (pair == null) {
        break;
      }
      String key = pair[0];
      String value = pair[1];
      out.printf("%s%s%s%n", key, separator, value);
    }
  }

  public static void main(String[] args) throws IOException {
    PrintWriter out = new PrintWriter(new OutputStreamWriter(System.out, "UTF-8"));
    if (args.length > 0) {
      for (String filename : args) {
        plist(out, filename);
      }
    }
    else {
      BufferedReader in = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
      while (true) {
        String filename = in.readLine();
        if (filename == null) {
          break;
        }
        plist(out, filename);
      }
    }
    out.flush();
    out.close();
  }

  public static void plist(PrintWriter out, String filename) throws IOException {
    out.printf("[ %s ]%n", filename);
    PListReader in = new PListReader(filename);
    in.printPList(out, " = ");
    in.close();
    out.println();
    // out.flush();
  }
}

$ cat DpmInfo.java

import java.io.File;
import java.io.IOException;

public class DpmInfo {
  private static final String pluginFolderName
    = "/Library/Application Support/Digidesign/Plug-ins";

  private String folderName;
  private String fileName;
  private PList plist;

  public DpmInfo(String dpmName) throws IOException {
    folderName = dpmName;
    if (! folderName.startsWith("/")) {
      folderName = pluginFolderName + "/" + folderName;
    }
    if (! folderName.endsWith(".dpm")) {
      folderName = folderName + ".dpm";
    }
    int n = folderName.lastIndexOf('/');
    fileName = folderName.substring(n+1);
    String plistFilename = folderName + "/Contents/Info.plist";
    PListReader in = new PListReader(plistFilename);
    plist = in.readPList();
    in.close();
  }

  public String getFolderName() {
    return folderName;
  }

  public String getFileName() {
    return fileName;
  }

  public String getBundleName() {
    return plist.get("CFBundleName");
  }

  public String getBundleVersion() {
    return plist.get("CFBundleVersion");
  }

  public String getBundleInfo() {
    String bundleInfo = plist.get("CFBundleGetInfoString");
    if (bundleInfo == null) {
      bundleInfo = "";
    }
    return bundleInfo;
  }
}

$ cat DpmLister.java

import java.io.File;
import java.io.IOException;

public class DpmLister {
  private static final int fontSize = 1;

  private static final String pluginFolderName
    = "/Library/Application Support/Digidesign/Plug-ins";

  public static void main(String[] args) throws IOException {
    File folder = new File(pluginFolderName);
    System.out.println("<table border=1>");
    System.out.print("<tr>");
    System.out.print("<td align=center><i>File</i></td>");
    System.out.print("<td align=center><i>Name</i></td>");
    System.out.print("<td align=center><i>Version</i></td>");
    System.out.print("<td align=center><i>Bundle Info</i></td>");
    System.out.println("</tr>");
    for (String dpmName : folder.list()) {
      if (! dpmName.endsWith(".dpm")) {
        continue;
      }
      DpmInfo dpmInfo = new DpmInfo(dpmName);
      printDpmInfo(dpmInfo);
    }
    System.out.println("</table>");
    System.out.flush();
  }

  public static void printDpmInfo(DpmInfo dpmInfo) throws IOException {
    String fileName = dpmInfo.getFileName();
    String bundleName = dpmInfo.getBundleName();
    String bundleVersion = dpmInfo.getBundleVersion();
    String bundleInfo = dpmInfo.getBundleInfo();
    if (bundleInfo == null) {
      bundleInfo = "";
    }
    print(new String[] { fileName, bundleName, bundleVersion, bundleInfo });
  }

  private static void print(String[] ss) {
    System.out.print("<tr>");
    for (String s : ss) {
      if (s == null || s.length() == 0) {
        s = "∓nbsp;";
      }
      System.out.printf("<td align=left valign=top><font size=%d>%s</font></td>", fontSize, s);
    }
    System.out.println("</tr>");
  }
}

$ javac *.java
$ java DpmLister > dpmlist.html
$ 

setRotation() {
    llSetRot(llEuler2Rot(<0.0, 0.0, PI>));
}

rezTriangles() {
    vector p = llGetPos();   
    llRezAtRoot("triangle", p + < 5.0, 0.0,      0.0>, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0); 
    llRezAtRoot("triangle", p + <-2.5, 4.330127, 0.0>, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
    llRezAtRoot("triangle", p + <-2.5,-4.330127, 0.0>, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
}  
 
default {
    state_entry() {
    }

    touch_start(integer total_number) {
        llRequestPermissions(llGetOwner(), PERMISSION_CHANGE_LINKS);
    }

    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            setRotation();
            rezTriangles();
        }
    }

    object_rez(key id) {
      llCreateLink(id, TRUE);
    }
} 

rezTriangle(vector pos, vector rot) {
    llRezAtRoot("large triangle", pos, <0.0, 0.0, 0.0>, llEuler2Rot(rot), 0); 
}

rezTriangles() {
    vector p = llGetPos();  
    rezTriangle(p + <-5.773502, 0.0, 4.08248>, <0.0, PI-0.95532, 0.0>); 
    rezTriangle(p + < 5.773502, 0.0, 4.08248>, <0.0,    0.95532, 0.0>);
    
    rezTriangle(p + <0.0,-5.773502, 4.08248>, <PI+0.95532, 0.0, PI_BY_TWO>); 
    rezTriangle(p + <0.0, 5.773502, 4.08248>, <PI-0.95532, 0.0,-PI_BY_TWO>);
}  

rezSquares() {
    rezSquaresX(llGetPos());
}

rezSquaresX(vector pos) {
    rezSquaresY(pos + <-4.330127, 0.0, 0.0>);
    rezSquaresY(pos + < 4.330127, 0.0, 0.0>);
}

rezSquaresY(vector pos) {
    rezSquare(pos + <0.0, -4.330127, 0.0>);
    rezSquare(pos + <0.0,  4.330127, 0.0>);
}

rezSquare(vector pos) {
    llRezObject("square 8.66m", pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
}

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
    }

    touch_start(integer total_number) {
        llRequestPermissions(llGetOwner(), PERMISSION_CHANGE_LINKS);
    }

    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezTriangles();
            rezSquares();
            setInvisible();
        }
    }

    object_rez(key id) {
      llCreateLink(id, TRUE);
    }
} 

rezTriangle(vector pos, vector rot) {
    llRezAtRoot("triangle", pos, <0.0, 0.0, 0.0>, llEuler2Rot(rot), 0); 
}

rezTriangles() {
    vector p = llGetPos();
    rezTriangle(p + <-2.88675, 0.0, 2.04124>, <0.0,   -0.95532, 0.0>); 
    rezTriangle(p + < 2.88675, 0.0, 2.04124>, <0.0, PI+0.95532, 0.0>);
    
    rezTriangle(p + <0.0,-2.88675, 2.04124>, < 0.95532, 0.0,  PI_BY_TWO>); 
    rezTriangle(p + <0.0, 2.88675, 2.04124>, <-0.95532, 0.0, -PI_BY_TWO>);
    
    rezTriangle(p + <-2.88675, 0.0, -2.04124>, <0.0,    0.95532, 0.0>); 
    rezTriangle(p + < 2.88675, 0.0, -2.04124>, <0.0, PI-0.95532, 0.0>);
    
    rezTriangle(p + <0.0,-2.88675, -2.04124>, <-0.95532, 0.0,  PI_BY_TWO>); 
    rezTriangle(p + <0.0, 2.88675, -2.04124>, < 0.95532, 0.0, -PI_BY_TWO>);
}

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}
 
default {
    state_entry() {
    }

    touch_start(integer total_number) {
        llRequestPermissions(llGetOwner(), PERMISSION_CHANGE_LINKS);
    }

    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezTriangles();
            setInvisible();
        }
    }

    object_rez(key id) {
      llCreateLink(id, TRUE);
    }
} 

rezTriangles() {
    vector p = llGetPos();
    llRezAtRoot("triangle", p + <-2.88675, 0.0, 2.04124>, <0.0, 0.0, 0.0>, llEuler2Rot(<0.0,   -0.95532, 0.0>), 0); 
    llRezAtRoot("triangle", p + < 2.88675, 0.0, 2.04124>, <0.0, 0.0, 0.0>, llEuler2Rot(<0.0, PI+0.95532, 0.0>), 0);
    
    llRezAtRoot("triangle", p + <0.0,-2.88675, 2.04124>, <0.0, 0.0, 0.0>, llEuler2Rot(< 0.95532, 0.0,  PI_BY_TWO>), 0); 
    llRezAtRoot("triangle", p + <0.0, 2.88675, 2.04124>, <0.0, 0.0, 0.0>, llEuler2Rot(<-0.95532, 0.0, -PI_BY_TWO>), 0);
}  
 
default {
    state_entry() {
    }

    touch_start(integer total_number) {
        llRequestPermissions(llGetOwner(), PERMISSION_CHANGE_LINKS);
    }

    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezTriangles();
        }
    }

    object_rez(key id) {
      llCreateLink(id, TRUE);
    }
} 

$ python
Python 2.5.1 (r251:54863, Apr 15 2008, 22:57:26) 
>>> from math import sqrt
>>> from math import atan2
>>> 5*sqrt(3)
8.6602540378443855
>>> 5*sqrt(6)/2
6.1237243569579451
>>> atan2( sqrt(6), sqrt(3) )
0.9553166181245093
>>> 

float height = 3776.0;

moveUp() {
    vector p = llGetPos();
    float h = p.z;
    while (h + 10.0 < height) {
        p += <0.0, 0.0, 10.0>;
        h += 10.0;
        llSetPos(p);
    }
    p.z = height;
    llSetPos(p);
}

default {
    state_entry() {
    }

    changed(integer param) {
        if (param & CHANGED_LINK) {
            moveUp();
            string name = llGetInventoryName(INVENTORY_OBJECT, 0);
            llRezObject(name, llGetPos(), <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
            llSetTimerEvent(6.0);
        }
    }

    timer() {
        llDie();
    }
} 

rezCube(string object, vector pos) {
    llRezObject(object, pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
}

rezCubes() {
    string name = "cube 10m ";
    vector pos = llGetPos();
    rezCubesX(name, pos);
}

rezCubesX(string name, vector pos) {
    rezCubesY(name + "-", pos + <-5.0, 0.0, 0.0>);
    rezCubesY(name + "+", pos + < 5.0, 0.0, 0.0>);
}

rezCubesY(string name, vector pos) {
    rezCubesZ(name + "-", pos + <0.0, -5.0, 0.0>);
    rezCubesZ(name + "+", pos + <0.0,  5.0, 0.0>);
}

rezCubesZ(string name, vector pos) {
    rezCube(name + "-", pos + <0.0, 0.0, -5.0>);
    rezCube(name + "+", pos + <0.0, 0.0,  5.0>);
}

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
    }

    touch_start(integer n) {
        key owner = llGetOwner();
        integer i;
        for (i = 0; i < n; i++) {
            if (llDetectedKey(i) == owner) {
                llRequestPermissions(owner, PERMISSION_CHANGE_LINKS);
            }
        }
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezCubes();
            setInvisible();
        }
    }
    
    object_rez(key id) {
        llCreateLink(id, TRUE);
    }
} 

vector size = <10.0, 10.0, 10.0>;

integer rezParam;

rezRectangle(string surface, vector pos, integer param) {
  llRezAtRoot(surface, pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, param);
}

rezRectangles() {
    string surface = llGetInventoryName(INVENTORY_OBJECT, 0);
    vector pos = llGetPos();
    if (rezParam & 1) {
        rezRectangle(surface, pos + <-size.x/2, 0.0, 0.0>, 1);
    }
    if (rezParam & 2) {
        rezRectangle(surface, pos + < size.x/2, 0.0, 0.0>, 1);
    }
    if (rezParam & 4) {
        rezRectangle(surface, pos + < 0.0,-size.y/2, 0.0>, 2);
    }
    if (rezParam & 8) {
        rezRectangle(surface, pos + < 0.0, size.y/2, 0.0>, 2);
    }
    if (rezParam & 16) {
        rezRectangle(surface, pos + < 0.0, 0.0,-size.z/2>, 3);
    }
    if (rezParam & 32) {
        rezRectangle(surface, pos + < 0.0, 0.0, size.z/2>, 3);
    }
}

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
    }

    touch_start(integer n) {
        if (rezParam > 0) {
            llRequestPermissions(llGetOwner(), PERMISSION_CHANGE_LINKS);
        }
    }
    
    on_rez(integer param) {
        rezParam = param;
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezRectangles();
            setInvisible();
        }
    }
    
    object_rez(key id) {
        llCreateLink(id, TRUE);
    }
} 

rezCube(string object, vector pos, integer param) {
    llRezObject(object, pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, param);
}

rezCubes() {
    string object = llGetInventoryName(INVENTORY_OBJECT, 0);
    vector pos = llGetPos();
    rezCubesX(object, pos, 0);
}

rezCubesX(string object, vector pos, integer param) {
    rezCubesY(object, pos + <-5.0, 0.0, 0.0>, param+1);
    rezCubesY(object, pos + < 5.0, 0.0, 0.0>, param+2);
}

rezCubesY(string object, vector pos, integer param) {
    rezCubesZ(object, pos + <0.0, -5.0, 0.0>, param+4);
    rezCubesZ(object, pos + <0.0,  5.0, 0.0>, param+8);
}

rezCubesZ(string object, vector pos, integer param) {
    rezCube(object, pos + <0.0, 0.0, -5.0>, param+16);
    rezCube(object, pos + <0.0, 0.0,  5.0>, param+32);
}

default {
    state_entry() {
    }

    touch_start(integer n) {
        key owner = llGetOwner();
        integer i;
        for (i = 0; i < n; i++) {
            if (llDetectedKey(i) == owner) {
                rezCubes();
            }
        }
    }
} 

vector size = <3.0, 4.0, 5.0>;

setSize(integer param) {
    vector v = size;
    if (param == 1) {
        v.x = 0.01;
    }
    else if (param == 2) {
        v.y = 0.01;
    }
    else if (param == 3) {
        v.z = 0.01;
    }
    else {
        return;
    }
    llSetScale(v);
}

default {
    state_entry() {
    }

    on_rez(integer param) {
        if (param > 0) {
            setSize(param);
        }
    }
} 

vector size = <3.0, 4.0, 5.0>;

key owner;

rezRectangle(vector pos, integer param) {
  llRezAtRoot("rectangle surface", pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, param);
}

rezRectangles() {
    vector pos = llGetPos();
    rezRectangle(pos + < size.x/2, 0.0, 0.0>, 1);
    rezRectangle(pos + <-size.x/2, 0.0, 0.0>, 1);
    rezRectangle(pos + < 0.0, size.y/2, 0.0>, 2);
    rezRectangle(pos + < 0.0,-size.y/2, 0.0>, 2);
    rezRectangle(pos + < 0.0, 0.0, size.z/2>, 3);
    rezRectangle(pos + < 0.0, 0.0,-size.z/2>, 3);
}

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
        owner = llGetOwner();
    }

    touch_start(integer n) {
        llRequestPermissions(owner, PERMISSION_CHANGE_LINKS);
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezRectangles();
            setInvisible();
        }
    }
    
    object_rez(key id) {
        llCreateLink(id, TRUE);
    }
} 

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
    }

    on_rez(integer param) {
        setInvisible();
    }
} 

string light = "light";

key owner;

setLightName() {
    if (llGetInventoryNumber(INVENTORY_OBJECT) > 0) {
        light = llGetInventoryName(INVENTORY_OBJECT, 0);
    }
}

rezLight(vector pos) {
    llRezObject(light, pos, <0.0, 0.0, 0.0>, ZERO_ROTATION, 0);
}

rezLights() {
    setLightName();
    vector pos = llGetPos();
    vector v = <0.10, 0.0, 0.0>;
    vector p = pos + v;
    rezLight(p);
}

default {
    state_entry() {
        owner = llGetOwner();
    }

    touch_start(integer n) {
        integer i;
        for (i = 0; i < n; i++) {
            if (llDetectedKey(i) == owner) {
                llRequestPermissions(owner, PERMISSION_CHANGE_LINKS);
            }
        }
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezLights();
        }
    }
    
    object_rez(key id) {
        llCreateLink(id, TRUE);
    }
} 

setInvisible() {
    llSetScale(<0.01, 0.01, 0.01>);
    llSetAlpha(0.0, ALL_SIDES);
}

default {
    state_entry() {
    }

    attach(key id) {
        setInvisible();
    }
} 

string square10 = "square 10m";
key owner;

rezObjects() {
    vector pos = llGetPos();
    integer i;
    for (i = -1; i < 2; i++) {
        integer j;
        for (j = 0; j < 2; j++) {
            if (i != 0 || j != 0) {
                llRezObject(square10, pos + <10.0 * (float)i, 10.0 * (float)j, 0.0>,
                            <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
            }
        }
    }
}

default {
    state_entry() {
        owner = llGetOwner();
    }

    touch_start(integer total_number) {
        integer i;
        for (i = 0; i < total_number; i++) {
            if (llDetectedKey(i) == owner) {
                llRequestPermissions(owner, PERMISSION_CHANGE_LINKS);
            }
        }
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezObjects();
        }
    }
    
    object_rez(key id) {
        llCreateLink(id, TRUE);
    }
} 

makeLargeSquare() {
    vector pos = llGetPos();
    llRezAtRoot("rectangle 30x20-1", pos + < 10.0,  10.0, 0.0>, <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
    llRezAtRoot("rectangle 20x30-2", pos + <-10.0,  10.0, 0.0>, <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
    llRezAtRoot("rectangle 30x20-3", pos + <-10.0, -10.0, 0.0>, <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
    llRezAtRoot("rectangle 20x30-4", pos + < 10.0, -10.0, 0.0>, <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
}

default {
    state_entry() {
    }
    
    on_rez(integer param) {
        makeLargeSquare();
    }
} 

default {
    state_entry() {
    }

    touch_start(integer n) {
        if (llDetectedKey(0) == llGetOwner()) {
            vector p = llGetPos();
            llRezObject("square 10m to 50m on rez", p + <0.0, 0.0, -1.0>,
                        <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);   
        }
    }
} 

string square10 = "square 10m";

key owner;
integer count = 8;
list squares = [];

rezSquares() {
    vector pos = llGetPos();
    integer i;
    for (i = -1; i < 2; i++) {
        integer j;
        for (j = -1; j < 2; j++) {
            if (i != 0 || j!= 0) {
                llRezAtRoot(square10, pos + <10.0 * (float)i, 10.0 * (float)j, 0.0>,
                                      <0.0, 0.0, 0.0>, <0.0, 0.0, 0,0>, 0);
            }
        }
    }
}

default {
    state_entry() {
        owner = llGetOwner();
    }

    touch_start(integer total_number) {
        integer i = 0;
        for ( ; i < total_number; i++) {
            if (llDetectedKey(i) == owner) {
                llRequestPermissions(owner, PERMISSION_CHANGE_LINKS);
            }
        }
    }
    
    run_time_permissions(integer perm) {
        if (perm & PERMISSION_CHANGE_LINKS) {
            rezSquares();
        }
    }
    
    object_rez(key id) {
        squares += id;
        count --;
        if (count == 0) {
            integer i;
            for (i = 0; i < 8; i++) {
                llCreateLink(llList2Key(squares, i), TRUE);
            }
        }
    }
} 

import java.io.*;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

public class KazMuzikServlet extends HttpServlet {
  private static final String filename = "kazmuzik.mp3";

  public void doGet(HttpServletRequest req, HttpServletResponse res)
      throws IOException, ServletException {

    File file = new File(filename);
    int len = (int)file.length();

    res.setContentType("audio/mpeg");
    res.setHeader("Content-Length", "" + len);
    res.setHeader("Content-Disposition",
                  "attachment; filename=\"" + filename + "\"");

    InputStream in = new BufferedInputStream(new FileInputStream(file));
    OutputStream out = new BufferedOutputStream(res.getOutputStream());
    byte[] b = new byte[len];
    int n = 0;
    while (true) {
      int m = in.read(b, n, len-n);
      out.write(b, n, m);
      n += m;
      if (n == len) {
        break;
      }
    }
    in.close();
    out.flush();
    out.close();
    return;
  }

  public void doPost(HttpServletRequest req, HttpServletResponse res)
      throws IOException, ServletException {
    doGet(req, res);
  }
} 

public class Test {
  private static final String text = "日本語のテスト";
  public static void main(String[] args) throws Exception {
    System.out.println(text);
  }
} 

$ echo $LANG
en_US.UTF-8
$ javac Test.java
$ java Test
日本語のテスト
$ 

$ mv Test.java Test.txt
$ native2ascii -encoding UTF-8 Test.txt > Test.java
$ grep final Test.java
  private static final String text = "\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8";
$ javac Test.java
$ java Test
???????
$ 

$ cat Test.java

public class Test {
  private static final String text = "\u65e5\u672c\u8a9e\u306e\u30c6\u30b9\u30c8";
  public static void main(String[] args) throws Exception {
    PrintWriter out = new PrintWriter(new OutputStreamWriter(System.out, "UTF-8"));
    out.println(text);
    out.flush();
  }
}

$ javac Test.java
$ java Test
日本語のテスト
$ 

integer defaultPrice = 100;
integer price = defaultPrice;
string item;

string notecardName;
integer notecardSize;

key owner;
key notecardRequest;

setInfo() {
    item = llGetInventoryName(INVENTORY_OBJECT, 0);
    if (item == "") {
        llInstantMessage(owner, "Please put an item to my inventory.");
    }
    else {
        notecardName = llGetInventoryName(INVENTORY_NOTECARD, 0);
        if (notecardName == "") {
            price = defaultPrice;
            llInstantMessage(owner, "The price is set to L$" + (string)defaultPrice
                    + " (default) because there is no notecard in my inventory.");
            setItemPrice();
        }
        else {
            integer value = (integer)notecardName;
            if (value > 0) {
                price = value;
                llInstantMessage(owner, "The price is set to L$" + (string)price
                    + " from the notecard name.");
                setItemPrice();
            }
            else {
                notecardSize = -1;
                notecardRequest = llGetNumberOfNotecardLines(notecardName);
                // -> dataserver()
            }
        }         
    }
}

setItemPrice() {
    llSetObjectName(item + " - item seller");
    llSetPayPrice(price, [price, PAY_HIDE, PAY_HIDE, PAY_HIDE]);
    llSetText(replaceName(item) + "\nL$" + (string)price, <1.0, 1.0, 1.0>, 1.0);
    string texture = llGetInventoryName(INVENTORY_TEXTURE, 0);
    if (texture != "") {
        llSetTexture(texture, ALL_SIDES);
    }
}

string replaceName(string s) {
    integer n = 0;
    while (n >= 0) {
        n = llSubStringIndex(s, " - ");
        if (n >= 0) {
            s = llDeleteSubString(s, n, n+2);
            s = llInsertString(s, n, "\n");
        }
    }
    return s;
}

               
default {
    state_entry() {
        owner = llGetOwner();
        llSetClickAction(CLICK_ACTION_PAY);
        setInfo();
    }
    
    changed(integer change) {
        if (change & CHANGED_INVENTORY) {
            setInfo();
        }
    }

    dataserver(key query, string data) {
        if (query == notecardRequest) {
            if (notecardSize < 0) {
                notecardSize = (integer)data;
                if (notecardSize <= 0) {
                    price = defaultPrice;
                    llInstantMessage(owner, "The price is set to L$" + (string)price
                        + " (default) because the notecard \"" + notecardName + "\" is empty.");
                    setItemPrice();
                }
                else {
                    notecardRequest = llGetNotecardLine(notecardName, 0);
                    // -> dataserver()
                }
            }
            else {
                price = (integer)data;
                if (price < 0) {
                    price = 0;
                    llInstantMessage(owner, "The price for \"" + item + "\" is set to L$" + (string)price
                        + " because the price of notecard \"" + notecardName + "\" is negative.");
                }
                else {
                    llInstantMessage(owner, "The price for \"" + item + "\" is set to L$" + (string)price
                        + " from the notecard \"" + notecardName + "\".");
                }
                setItemPrice();
            }
        }
    }
    
    money(key buyer, integer amount) {
        if (amount >= price) {
            llInstantMessage(owner, llKey2Name(buyer) + " paid you L$" + (string)amount + " for \"" + item + "\".");
            llInstantMessage(buyer, "Thank you for your purchase of \"" + item + "\" with L$" + (string)amount + ".");
            llGiveInventory(buyer, item);
        }
        else {
            llInstantMessage(buyer, "You paid L$" + (string)amount + ", but it is less than the price L$"
                    + (string)price + ", so your payment is considered as a donation. Thank you for your donation.");
            llInstantMessage(owner, llKey2Name(buyer) + " paid you L$" + (string)amount + " as donation.");
        }
    }
} 

integer defaultPrice = 100;
integer price = defaultPrice;

string notecardName;
integer notecardSize;

key owner;

key notecardRequest;

setPrice() {
    notecardName = llGetInventoryName(INVENTORY_NOTECARD, 0);
    if (notecardName == "") {
        price = defaultPrice;
        llInstantMessage(owner, "The price is set to L$" + (string)defaultPrice
                     + " because there is no notecard in my inventory.");
        llSetPayPrice(price, [price, PAY_HIDE, PAY_HIDE, PAY_HIDE]);
    }
    else {
        notecardSize = -1;
        notecardRequest = llGetNumberOfNotecardLines(notecardName);
        // -> dataserver()               
    }
}
               
default {
    state_entry() {
        owner = llGetOwner();
        llSetClickAction(CLICK_ACTION_PAY);
        setPrice();
    }
    
    changed(integer change) {
        if (change & CHANGED_INVENTORY) {
            setPrice();
        }
    }

    dataserver(key query, string data) {
        if (query == notecardRequest) {
            if (notecardSize < 0) {
                notecardSize = (integer)data;
                if (notecardSize <= 0) {
                    price = defaultPrice;
                    llInstantMessage(owner, "The price is set to L$" + (string)defaultPrice
                        + " because the notecard \"" + notecardName + "\" is empty.");
                    llSetPayPrice(price, [price, PAY_HIDE, PAY_HIDE, PAY_HIDE]);
                }
                else {
                    notecardRequest = llGetNotecardLine(notecardName, 0);
                    // -> dataserver()
                }
            }
            else {
                price = (integer)data;
                if (price < 0) {
                    price = 0;
                    llInstantMessage(owner, "The price is set to L$" + (string)price
                        + " because the price of notecard \"" + notecardName + "\" is negative.");
                }
                else {
                    llInstantMessage(owner, "The price is set to L$" + (string)price
                        + " from the notecard \"" + notecardName + "\".");
                }
                llSetPayPrice(price, [price, PAY_HIDE, PAY_HIDE, PAY_HIDE]);
            }
        }
    }
    
    money(key payer, integer amount) {
        llInstantMessage(owner, llKey2Name(payer) + " paid you L$" + (string)amount + ".");
    }
} 

[13:01]  Payment Example: The price is set to L$100 because there is no notecard in my inventory.
[13:02]  Payment Example: The price is set to L$10 from the notecard "sample price".
[13:03]  Payment Example: The price is set to L$1 from the notecard "sample price".
[13:04]  Payment Example: Sample Payer paid you L$1.

string notecardName;
integer notecardSize;
integer notecardRead;
list notecardLines;

key owner;

key notecardRequest;

setNotecard() {
     notecardName = llGetInventoryName(INVENTORY_NOTECARD, 0);
}

printNotecardName() {
    llInstantMessage(owner, "notecardName=\"" + notecardName + "\"");
}

printNotecardLines() {
    string s = "";
    s += "\n---BEGIN OF NOTECARD ---";
    integer i = 0;
    for ( ; i < notecardSize; i++) {
        s += ("\n" + llList2String(notecardLines, i));
    }
    s += "\n--- END OF NOTECARD ---";
    llInstantMessage(owner, s);
}
                    
default {
    state_entry() {
        setNotecard();
        owner = llGetOwner();
        llSetTouchText("Read Notecard");
        printNotecardName();
    }
    
    changed(integer change) {
        if (change & CHANGED_INVENTORY) {
            setNotecard();
            printNotecardName();
        }
    }

    touch_start(integer total_number) {
        integer i = 0;
        for ( ; i < total_number ; i++) {
            key avator = llDetectedKey(i);
            if (avator == owner) {
                if (notecardName == "") {
                    llInstantMessage(owner, "no notecard in my inventory");
                }
                else {
                    notecardSize = -1;
                    notecardRequest = llGetNumberOfNotecardLines(notecardName);               
                }
            }
        }
    }
    
    dataserver(key query, string data) {
        if (query == notecardRequest) {
            if (notecardSize < 0) {
                notecardSize = (integer)data;
                if (notecardSize <= 0) {
                    llInstantMessage(owner, "The notecard \"" + notecardName + "\" is empty.");
                }
                else {
                    llInstantMessage(owner, (string)notecardSize + " lines"
                             + " in the notecard \"" + notecardName + "\".");
                    notecardRead = 0;
                    notecardLines = [];
                    notecardRequest = llGetNotecardLine(notecardName, 0);
                }
            }
            else {
                notecardRead ++;
                notecardLines += data;
                if (notecardRead < notecardSize) {
                    notecardRequest = llGetNotecardLine(notecardName, notecardRead);
                }
                else {
                    printNotecardLines();
                }
            }
        }
    }
} 

[10:10]  Notecard example: notecardName="sample text"
[10:11]  Notecard example: 3 lines in the notecard "sample text".
[10:11]  Notecard example: 
---BEGIN OF NOTECARD ---
This is the first line.
This is the second line.
これは、3行目です。
--- END OF NOTECARD ---

key httpRequestId;
key owner;
 
default {
   state_entry() {
        llSay(0, "Hello, Avatar!");
    }

    touch_start(integer total_number) {
        key obj = llGetKey();
        owner = llGetOwner();
        key creator = llGetCreator();
        string objName = llKey2Name(obj);
        string ownerName = llKey2Name(owner);
        string creatorName = llKey2Name(creator);
        integer i = 0;
        for ( ; i < total_number; i++) {
            key avator = llDetectedKey(i);
            if (avator != owner && avator != creator) {
                llInstantMessage(avator, "Sorry, you cannot touch me.");
            }
            else {
                llInstantMessage(avator,
                  "I am " + objName + " (" + (string)obj + ").");
                llInstantMessage(avator,
                  "I am owned by " + ownerName + " (" + (string)owner + ").");
                llInstantMessage(avator,
                  "I was created by " + creatorName + " (" + (string)creator + ").");
                llInstantMessage(avator,
                  "You are " + llDetectedName(i) + " (" + (string)avator + ").");
                httpRequestId = llHTTPRequest("http://kazmuzik.net:8080/sltest/test.jsp", [], "");
            }
        }
    }
       
    http_response(key requestId, integer status, list metadata, string body) {
        if (requestId == httpRequestId) {
            llInstantMessage(owner, llStringTrim(body, STRING_TRIM));
        }
    }
} 

<%
    String objectKey  = request.getHeader("X-SecondLife-Object-Key");
    String objectName = request.getHeader("X-SecondLife-Object-Name");
    String ownerKey   = request.getHeader("X-SecondLife-Owner-Key");
    String ownerName  = request.getHeader("X-SecondLife-Owner-Name");
    java.util.Formatter f = new java.util.Formatter();
    f.format("%s (%s) touched by %s (%s)%n", objectName, objectKey, ownerName, ownerKey);
%>
<%= f.toString() %>

$ cd /opt/tomcat/webapps
$ mkdir sltest
$ cd sltest
$ vi test.jsp (上記のファイル)
$

[01:23]  HTTP sample object: I am HTTP sample object (12345678-90ab-cdef-1234-567890abcdef).
[01:23]  HTTP sample object: I am owned by Kaz Muzik (abcdef12-3456-7890-abcd-ef1234567890).
[01:23]  HTTP sample object: I was created by Kaz Muzik (abcdef12-3456-7890-abcd-ef1234567890).
[01:23]  HTTP sample object: You are Kaz Muzik (abcdef12-3456-7890-abcd-ef1234567890).
[01:23]  HTTP sample object: HTTP sample object (12345678-90ab-cdef-1234-567890abcdef) touched \
                             by Kaz Muzik (abcdef12-3456-7890-abcd-ef1234567890)

default {
    state_entry() {
        llSay(0, "Hello, Avatar!");
    }

    touch_start(integer total_number) {
        llSay(0, "Touched.");
        key k = llGetKey();
        llSay(0, "My key is " + (string)k + ".");
        llSay(0, "My name is " + llKey2Name(k) + ".");
        k = llGetOwner();
        llSay(0, "My owner's key is " + (string)k + ".");
        llSay(0, "My owner's name is " + llKey2Name(k) + ".");
        integer i = 0;
        for ( ; i < total_number; i++) {
            k = llDetectedKey(i);
            llInstantMessage(k, "Your key is " + (string)k + ".");
            llInstantMessage(k, "Your name is " + llDetectedName(i) + ".");
        }
    }
} 

Object: Touched.
Object: My key is 12345678-90ab-cdef-1234-567890abcdef.
Object: My name is Object.
Object: My owner's key is abcdef12-3456-7890-abcd-ef1234567890.
Object: My owner's name is Kaz Muzik.
Object: Your key is abcdef12-3456-7890-abcd-ef1234567890.
Object: Your name is Kaz Muzik.

# yum install ruby
# yum install rubygems
$ rpm -q ruby
ruby-1.8.6.230-4.fc8
$ rpm -q rubygems
rubygems-0.9.4-1.fc8
# gem install rake
# gem install raven
#

$ ls -F /usr/lib/ruby/gems/1.8/gems/raven-1.2.4
bin/  lib/  LICENSE  README.rdoc
$ rake -V
rake, version 0.8.1
$ raven import
$ ls -F ~/.raven
cache/  doc/  gems/  m2_central.mvnidx  m2_ibiblio.mvnidx  specifications/
$

$ cd ~/kazmuzik-pagerank
$ cat rakefile

require 'raven'
require 'rake/clean'

CLEAN.include('target')

task 'default' => 'kazmuzik-pagerank.jar'

dependency 'compile_deps' do |t| 
    t.deps << ['commons-math']
end

javac 'compile' => 'compile_deps' do |t|
    t.build_path << "src/main/java"
end

jar 'kazmuzik-pagerank.jar' => 'compile'

gem_wrap_inst 'gem' => 'kazmuzik-pagerank.jar' do |t|
    t.version = '1.0'
end

$ rake kazmuzik-pagerank.jar --trace
(in /home/kazmuzik/kazmuzik-pagerank)
** Invoke kazmuzik-pagerank.jar (first_time)
** Invoke compile (first_time)
** Invoke compile_deps (first_time)
rake aborted!
wrong number of arguments (1 for 0)
...
$ 

# gem update --system
# yum install ruby-devel
# gem install rcov
$ rake
(in /home/kazmuzik/kazmuzik-pagerank)
rake aborted!
Platform is not a module
/usr/lib/ruby/gems/1.8/gems/raven-1.2.4/lib/raven/gem_init.rb:22
...
$ 

$ wget http://rubyforge-files.ruby-forum.com/raven/raven-1.2.4-jruby-1.0.1.zip
$ cd /opt
$ unzip ~/raven-1.2.4-jruby-1.0.1.zip
$ ln -s raven-1.2.4-jruby-1.0.1 raven
$ PATH=/opt/raven/bin:$PATH
$ cd ~/kazmuzik-pagerank
$ rake clean
(in /home/kazmuzik/kazmuzik-pagerank)
rm -r target
$ rake
(in /home/kazmuzik/kazmuzik-pagerank)
Using local gem commons-math-commons-math (1.2) to satisfy dependency commons-math
Building path src/main/java
javac -classpath "/home/kazmuzik/.raven/gems/commons-math-commons-math-1.2-java/ext/commons-math-1.2.jar\
:target/classes" -sourcepath "src/main/java" -d target/classes src/main/java/net/kazmuzik/pagerank/*.java
Built jar file kazmuzik-pagerank.jar.
$ ls -F target
classes/  kazmuzik-pagerank.jar
$ 

% sudo yum -y install maven2
...
$ mvn -v
/usr/lib/jvm/java
Maven version: 2.0.4
$ export JAVA_HOME=/usr/java/jdk
$ mvn -v
/usr/java/jdk
Maven version: 2.0.4
$ 

$ mvn archetype:create -DgroupId=net.kazmuzik.pagerank -DartifactId=kazmuzik-pagerank
...
[INFO] artifact org.apache.maven.plugins:maven-archetype-plugin: checking for updates from central
Downloading: http://repo1.maven.org/maven2/org/apache/maven/plugins/maven-archetype-plugin/2.0-alpha-3/\
maven-archetype-plugin-2.0-alpha-3.pom 
4K downloaded
[WARNING] Skipping jpp repository file:///usr/share/maven2/repository in vanilla mode
...
$ ls -l /usr/share/maven2/repository
lrwxrwxrwx 1 root root 15 2008-08-20 23:45 JPP -> /usr/share/java
$ 

$ cat pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>net.kazmuzik.pagerank</groupId>
  <artifactId>kazmuzik-pagerank</artifactId>
  <packaging>jar</packaging>
  <version>1.0-SNAPSHOT</version>
  <name>kazmuzik-pagerank</name>
  <url>http://maven.apache.org</url>
  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>commons-math</groupId>
      <artifactId>commons-math</artifactId>
      <version>1.2</version>
      <scope>compile</scope>
    </dependency>
  </dependencies>
  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>1.5</source>
          <target>1.5</target>
        </configuration>
      </plugin>
    </plugins>
  </build>
</project>
$ mvn compile
...
$ mvn package
...
$ ls -F target
classes/  kazmuzik-pagerank-1.0-SNAPSHOT.jar  surefire-reports/  test-classes/
$ ls ~/.m2/repository/commons-math/commons-math/1.2
commons-math-1.2.jar      commons-math-1.2.pom
commons-math-1.2.jar.md5  commons-math-1.2.pom.md5
$ 

$ java -classpath target/kazmuzik-pagerank-1.0-SNAPSHOT.jar:\
/home/kazmuzik/.m2/repository/commons-math/commons-math/1.2/commons-math-1.2.jar \
 net.kazmuzik.pagerank.PageRankExample1
A       0.0303
B       0.3854
C       0.3438
D       0.0392
E       0.0811
F       0.0392
g       0.0162
h       0.0162
i       0.0162
j       0.0162
k       0.0162
$ 

import org.apache.commons.math.linear.MatrixUtils;
import org.apache.commons.math.linear.RealMatrix;

public class PageRanker {

  ...

  public double[] solve() {
    int len = pages.length;
    double[][] a = new double[len][len];
    for (int j = 0; j < len; j++) {
      int n = 0;
      for (int i = 0; i < len; i++) {
        n += links[i][j];
      }
      double v = -1. * dampingFactor / (double)(n==0?(len-1):n);
      for (int i = 0; i < len; i++) {
        a[i][j] = (i==j)?1.:((double)(n==0?1:links[i][j])*v);
      }
    }
    if (dampingFactor == 1.) {
      for (int j = 0; j < len; j++) {
        a[0][j] = 1.;
      }
    }
    RealMatrix A = MatrixUtils.createRealMatrix(a);

    double[] b = new double[len];
    if (dampingFactor == 1.) {
      b[0] = 1.;
      for (int i = 1; i < len; i++) {
        b[i] = 0.;
      }
    }
    else {
      double v = (1. - dampingFactor) / (double)len;
      for (int i = 0; i < len; i++) {
        b[i] = v;
      }
    }
    RealMatrix B = MatrixUtils.createRowRealMatrix(b).transpose();

    RealMatrix X = A.solve(B);

    double[] pageRanks = new double[len];
    for (int i = 0; i < len; i++) {
      pageRanks[i] = X.getEntry(i, 0);
    }
    return pageRanks;
  }
} 

$ javac -classpath commons-math-1.2.jar PageRanker.java
$ java -classpath .:commons-math-1.2.jar PageRankExample1
A       0.0303
B       0.3854
C       0.3438
D       0.0392
E       0.0811
F       0.0392
g       0.0162
h       0.0162
i       0.0162
j       0.0162
k       0.0162
$ java -classpath .:commons-math-1.2.jar PageRankExample2
ID=1    0.3035
ID=2    0.1661
ID=3    0.1406
ID=4    0.1054
ID=5    0.1789
ID=6    0.0447
ID=7    0.0607
$ 

public class PageRankExample2 extends PageRanker {

  public PageRankExample2() {
    super( new String[] {"ID=1", "ID=2", "ID=3", "ID=4",
                         "ID=5", "ID=6", "ID=7"} );
    addLinks();
    setDampingFactor(1.0);
  }

  private void addLinks() {
    addLink("ID=1", "ID=2");
    addLink("ID=1", "ID=3");
    addLink("ID=1", "ID=5");
    addLink("ID=1", "ID=6");
    addLink("ID=2", "ID=1");
    addLink("ID=2", "ID=3");
    addLink("ID=2", "ID=4");
    addLink("ID=3", "ID=1");
    addLink("ID=3", "ID=4");
    addLink("ID=3", "ID=5");
    addLink("ID=4", "ID=1");
    addLink("ID=4", "ID=5");
    addLink("ID=5", "ID=1");
    addLink("ID=5", "ID=4");
    addLink("ID=5", "ID=6");
    addLink("ID=5", "ID=7");
    addLink("ID=6", "ID=5");
    addLink("ID=7", "ID=1");
  }

  public static void main(String[] args) throws Exception {
    PageRanker pr = new PageRankExample2();
    double[] ranks = pr.solve();
    print(pr.getPages(), ranks);
  }

  private static void print(String[] pages, double[] ranks) {
    int i = 0;
    for (String page : pages) {
      print(page, ranks[i++]);
    }
  }

  private static void print(String page, double rank) {
    System.out.printf("%s\t%.4f%n", page, rank);
  }
} 

$ java -classpath .:Jama-1.0.2.jar PageRankExample2

  1.000 -1.000 -0.500 -0.000 -0.250 -0.500 -0.000
 -0.200  1.000 -0.500 -0.333 -0.000 -0.000 -0.000
 -0.200 -0.000  1.000 -0.333 -0.250 -0.000 -0.000
 -0.200 -0.000 -0.000  1.000 -0.250 -0.000 -0.000
 -0.200 -0.000 -0.000 -0.333  1.000 -0.500 -1.000
 -0.000 -0.000 -0.000 -0.000 -0.250  1.000 -0.000
 -0.200 -0.000 -0.000 -0.000 -0.000 -0.000  1.000

Exception in thread "main" java.lang.RuntimeException: Matrix is singular.
        at Jama.LUDecomposition.solve(LUDecomposition.java:282)
        at Jama.Matrix.solve(Matrix.java:815)
        at PageRanker.solve(PageRanker.java:59)
        at PageRankExample2.main(PageRankExample2.java:33)
$ 

public class PageRanker {
  ...
  public double[] solve() {
    int len = pages.length;
    Matrix A = new Matrix(len,len);
    for (int j = 0; j < len; j++) {
      int n = 0;
      for (int i = 0; i < len; i++) {
        n += links[i][j];
      }
      double v = -1. * dampingFactor / (double)(n==0?(len-1):n);
      for (int i = 0; i < len; i++) {
        A.set(i, j, (i==j)?1.:((double)(n==0?1:links[i][j])*v));
      }
    }
    if (dampingFactor == 1.) {
      for (int j = 0; j < len; j++) {
        A.set(0, j, 1.);
      }
    }
    A.print(5,3);

    Matrix B = new Matrix(len,1);
    if (dampingFactor == 1.) {
      B.set(0, 0, 1.);
      for (int i = 1; i < len; i++) {
        B.set(i, 0, 0.);
      }
    }
    else {
      double v = (1. - dampingFactor) / (double)len;
      for (int i = 0; i < len; i++) {
        B.set(i, 0, v);
      }
    }   

    Matrix X = A.solve(B);

    double[] pageRanks = new double[len];
    for (int i = 0; i < len; i++) {
      pageRanks[i] = X.get(i, 0);
    }
    return pageRanks;
  }
} 

R = B - A*R

(I-A)R = B

import java.util.HashMap;
import java.util.Map;
import Jama.Matrix;

public class PageRanker {
  private String[] pages;
  private int[][] links;
  private Map<String,Integer> pageMap;
  private double dampingFactor = 0.85;

  public PageRanker(String[] pages) {
    this.pages = pages;
    links = new int[pages.length][pages.length];
    pageMap = new HashMap<String,Integer>();
    int i = 0;
    for (String page : pages) {
      pageMap.put(page, i++);
    }
  }

  public String[] getPages() {
    return pages;
  }

  public void addLink(String pageTo, String pageFrom) {
    int i = pageMap.get(pageTo);
    int j = pageMap.get(pageFrom);
    links[i][j] += 1;
  }

  public void setDampingFactor(double d) {
    dampingFactor = d;
  }

  public double[] solve() {
    int len = pages.length;
    Matrix A = new Matrix(len,len);
    for (int j = 0; j < len; j++) {
      int n = 0;
      for (int i = 0; i < len; i++) {
        n += links[i][j];
      }
      double v = -1. * dampingFactor / (double)(n==0?(len-1):n);
      for (int i = 0; i < len; i++) {
        A.set(i, j, (i==j)?1.:((double)(n==0?1:links[i][j])*v));
      }
    }
    A.print(5,3);

    Matrix B = new Matrix(len,1);
    double v = (1. - dampingFactor) / (double)len;
    for (int i = 0; i < len; i++) {
      B.set(i, 0, v);
    }

    Matrix X = A.solve(B);

    double[] pageRanks = new double[len];
    for (int i = 0; i < len; i++) {
      pageRanks[i] = X.get(i, 0);
    }
    return pageRanks;
  }
}

public class PageRankExample1 extends PageRanker {

  public PageRankExample1() {
    super( new String[] {"A", "B", "C", "D", "E", "F",
                         "g", "h", "i", "j", "k"} );
    addLinks();
  }

  private void addLinks() {
    addLink("A", "D");
    addLink("B", "C");
    addLink("B", "D");
    addLink("B", "E");
    addLink("B", "F");
    addLink("B", "g");
    addLink("B", "h");
    addLink("B", "i");
    addLink("C", "B");
    addLink("D", "E");
    addLink("E", "F");
    addLink("E", "g");
    addLink("E", "h");
    addLink("E", "i");
    addLink("E", "j");
    addLink("E", "k");
    addLink("F", "E");
  }

  public static void main(String[] args) throws Exception {
    PageRanker pr = new PageRankExample1();
    double[] ranks = pr.solve();
    print(pr.getPages(), ranks);
  }

  private static void print(String[] pages, double[] ranks) {
    int i = 0;
    for (String page : pages) {
      print(page, ranks[i++]);
    }
  }

  private static void print(String page, double rank) {
    System.out.printf("%s\t%.4f%n", page, rank);
  }
}

$ javac -classpath Jama-1.0.2.jar PageRanker.java PageRankExample1.java
$ java -classpath .:Jama-1.0.2.jar PageRankExample1

  1.000 -0.000 -0.000 -0.425 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000
 -0.085  1.000 -0.850 -0.425 -0.283 -0.425 -0.425 -0.425 -0.425 -0.000 -0.000
 -0.085 -0.850  1.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000
 -0.085 -0.000 -0.000  1.000 -0.283 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000
 -0.085 -0.000 -0.000 -0.000  1.000 -0.425 -0.425 -0.425 -0.425 -0.850 -0.850
 -0.085 -0.000 -0.000 -0.000 -0.283  1.000 -0.000 -0.000 -0.000 -0.000 -0.000
 -0.085 -0.000 -0.000 -0.000 -0.000 -0.000  1.000 -0.000 -0.000 -0.000 -0.000
 -0.085 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000  1.000 -0.000 -0.000 -0.000
 -0.085 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000  1.000 -0.000 -0.000
 -0.085 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000  1.000 -0.000
 -0.085 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000 -0.000  1.000

A       0.0303
B       0.3854
C       0.3438
D       0.0392
E       0.0811
F       0.0392
g       0.0162
h       0.0162
i       0.0162
j       0.0162
k       0.0162

$ 

 x + y = 12 ... (1)

 2 * x + 4 * y = 34 ... (2)

 1 1     x     12
(   ) * ( ) = (  )  ... (3)
 2 4     y     34

$ cat CraneAndTortoise.java
import Jama.Matrix;

public class CraneAndTortoise {
  public static void main(String[] args) throws Exception {
    int n = Integer.parseInt(args[0]);
    int m = Integer.parseInt(args[1]);

    Matrix A = new Matrix( new double[][] {{1.,1.},{2.,4.}} );
    print(A);

    Matrix B = new Matrix( new double[][] {{(double)n},{(double)m}} );
    print(B);

    Matrix X = A.solve(B);
    print(X);
  }

  private static void print(Matrix M) {
    M.print(2,0);
  }
}
$ wget http://math.nist.gov/javanumerics/jama/Jama-1.0.2.jar
$ javac -classpath Jama-1.0.2.jar CraneAndTortoise.java
$ java -classpath .:Jama-1.0.2.jar CraneAndTortoise 12 34

   1   1
   2   4


  12
  34


   7
   5

$ 

import java.io.InputStreamReader;
import java.util.Arrays;
import au.com.bytecode.opencsv.CSVReader;

public class CSVReaderTest {
  public static void main(String[] args) throws Exception {
    CSVReader in = new CSVReader(new InputStreamReader(System.in, "UTF-8"));
    while (true) {
      String[] values = in.readNext();
      if (values == null) {
        break;
      }
      System.out.println( Arrays.toString(values) );
    }
    in.close();
  }
} 

$ cat lj.sh
#!/bin/sh

L=$SAMPO_HOME/lib
CP=$L/sampo-nutch.jar:$L/hadoop-0.12.2-core.jar:$L/nutch-0.9.jar\
:$L/commons-logging-1.1.1.jar:$L/log4j-1.2.15.jar:$L/commons-io-1.3.2.jar
CN=net.java.sampo.nutch.util.ContentsExtractor

S=`ls -d kazmuzik-segment-$1/20* | tail -1`
U="http://kazuomik.livejournal.com/"
D=$2

java -classpath $CP $CN $S $U $D

$ sh lj.sh 20080309 lj
$ 

$ cvs -d :pserver:username@cvs.dev.java.net:/cvs login
$ cvs -d :pserver:username@cvs.dev.java.net:/cvs checkout sampo
$ cd sampo/nutch
$ export JAVA_HOME=/usr/java/jdk1.6.0_04
$ export SAMPO_HOME=/var/sampo
$ export NUTCH_HOME=/usr/local/nutch-0.9
$ export PATH=$SAMPO_HOME/bin:$JAVA_HOME/bin:$PATH
$ ant init
$ ant -DSAMPO_HOME=$SAMPO_HOME install
$ cd $NUTCH_HOME
$ mkdir lj
$ vi lj.sh
$ cat lj.sh
#!/bin/sh

L=$SAMPO_HOME/lib
CP=$L/sampo-nutch.jar:$L/hadoop-0.12.2-core.jar:$L/nutch-0.9.jar\
:$L/commons-logging-1.1.1.jar:$L/log4j-1.2.15.jar:$L/commons-io-1.3.2.jar
CN=net.java.sampo.nutch.util.ContentsExtractor

S=`ls -d $1/20* | tail -1`
D=$2

java -classpath $CP $CN $S $D

$ sh lj.sh kazmuzik-segment-20080303 lj
$ 

$ cvs -d :pserver:username@cvs.dev.java.net:/cvs login
$ cvs -d :pserver:username@cvs.dev.java.net:/cvs checkout sampo
$ cd sampo/language
$ export JAVA_HOME=/usr/java/jdk1.6.0_04
$ export SAMPO_HOME=/var/sampo
$ export PATH=$SAMPO_HOME/bin:$JAVA_HOME/bin:$PATH
$ ant init
$ ant compile
$ ant -DSAMPO_HOME=$SAMPO_HOME install
$ cd $SAMPO_HOME
$ rm -rf wikipedia-mainpage-text LM
$ wikipedia-mainpage-text.sh 10000 2
$ create-local-encoding-texts.sh
$ wikipedia-mainpage-createfp.sh
$ cd wikipedia-mainpage-text
$ test-jni-libtextcat.sh en.*.txt ja.*.txt
en.ISO-8859-15.txt      [en.ISO-8859-15]
en.UTF-8.txt    [en.ISO-8859-15][en.UTF-8]
ja.EUC-JP.txt   [ja.EUC-JP]
ja.ISO-2022-JP.txt      [ja.ISO-2022-JP]
ja.Shift_JIS.txt        [ja.Shift_JIS]
ja.UTF-8.txt    [ja.UTF-8]
ja.EUC-JP.txt   [ja.EUC-JP]
ja.ISO-2022-JP.txt      [ja.ISO-2022-JP]
ja.Shift_JIS.txt        [ja.Shift_JIS]
ja.UTF-8.txt    [ja.UTF-8]
$ 

$ test-jni-libtextcat.sh ja.*.txt ja.*.txt en.*.txt zh.*.txt
ja.EUC-JP.txt   [ja.EUC-JP]
ja.ISO-2022-JP.txt      [ja.ISO-2022-JP]
ja.Shift_JIS.txt        [ja.Shift_JIS]
ja.UTF-8.txt    [ja.UTF-8]
ja.EUC-JP.txt   [ja.EUC-JP]
ja.ISO-2022-JP.txt      [ja.ISO-2022-JP]
ja.Shift_JIS.txt        [ja.Shift_JIS]
ja.UTF-8.txt    [ja.UTF-8]
en.ISO-8859-15.txt      [en.ISO-8859-15]
en.UTF-8.txt    [en.ISO-8859-15][en.UTF-8]
zh.Big5.txt     [zh.Big5]
zh.GB18030.txt  [zh.GB18030]
zh.GB2312.txt   [zh.GB2312]
zh.UTF-8.txt    [zh.UTF-8]
$ 

# yum install libtextcat-devel
$ rpm -q libtextcat-devel
libtextcat-devel-2.2-4.fc8
libtextcat-devel-2.2-4.fc8
$ rpm -ql libtextcat-devel
/usr/bin/createfp
/usr/include/libtextcat
/usr/include/libtextcat/common.h
/usr/include/libtextcat/constants.h
/usr/include/libtextcat/fingerprint.h
/usr/include/libtextcat/textcat.h
/usr/include/libtextcat/utf8misc.h
/usr/lib64/libtextcat.so
/usr/bin/createfp
/usr/include/libtextcat
...
/usr/lib/libtextcat.so
$ 

$ cvs -d :pserver:username@cvs.dev.java.net:/cvs login
$ cvs -d :pserver:username@cvs.dev.java.net:/cvs checkout sampo
$ cd sampo/language
$ export JAVA_HOME=/usr/java/jdk1.6.0_04
$ export SAMPO_HOME=/var/sampo
$ export PATH=$SAMPO_HOME/bin:$JAVA_HOME/bin:$PATH
$ ant init
$ ant compile
$ sh bin/javah.sh
$ sh bin/jnicc.sh
$ ant -DSAMPO_HOME=$SAMPO_HOME install
$ test-jni-libtextcat.sh ja.*.txt en.*.txt zh.*.txt
ja.EUC-JP.txt   
ja.ISO-2022-JP.txt      [ja.ISO-2022-JP]
ja.Shift_JIS.txt        [ja.Shift_JIS]
ja.UTF-8.txt    [ja.UTF-8]
en.ISO-8859-15.txt      [en.ISO-8859-15]
en.UTF-8.txt    [en.UTF-8]
zh.Big5.txt     
zh.GB18030.txt  
zh.GB2312.txt   [zh.GB2312]
zh.UTF-8.txt    [zh.UTF-8]
$ 

en.ISO-8859-15
es.ISO-8859-15
de.ISO-8859-15
fr.ISO-8859-15
it.ISO-8859-15
po.ISO-8859-15
cs.ISO-8859-2
sl.ISO-8859-2
bg.ISO-8859-5
ru.ISO-8859-5
ar.ISO-8859-6
el.ISO-8859-7
he.ISO-8859-8
ja.EUC-JP
ko.EUC-KR
ja.ISO-2022-JP
ko.ISO-2022-KR
ja.Shift_JIS
zh.GB18030
zh.GB2312
zh.Big5
ru.KOI8-R
uk.KOI8-R

$ create-local-encoding-texts.sh
$ wikipedia-mainpage-createfp.sh
$ cd $SAMPO_HOME/wikipedia-mainpage-text
$ testtextcat.sh en.*.txt
en.ISO-8859-15.txt
TextCat 2.2 (out of place)
Result == [en.ISO-8859-15]
That took 5 ms.
en.UTF-8.txt
TextCat 2.2 (out of place)
Result == [en.ISO-8859-15][en.UTF-8]
That took 5 ms.
en.ISO-8859-15.txt
TextCat 2.2 (out of place)
Result == [en.ISO-8859-15]
That took 5 ms.
en.UTF-8.txt
TextCat 2.2 (out of place)
Result == [en.ISO-8859-15][en.UTF-8]
That took 5 ms.
$ testtextcat.sh ja.*.txt
ja.EUC-JP.txt
TextCat 2.2 (out of place)
Result == [ja.EUC-JP]
That took 3 ms.
ja.ISO-2022-JP.txt
TextCat 2.2 (out of place)
Result == [ja.ISO-2022-JP]
That took 4 ms.
ja.Shift_JIS.txt
TextCat 2.2 (out of place)
Result == [ja.Shift_JIS]
That took 4 ms.
ja.UTF-8.txt
TextCat 2.2 (out of place)
Result == [ja.UTF-8]
That took 3 ms.
$ testtextcat.sh zh*.txt
zh.Big5.txt
TextCat 2.2 (out of place)
Result == [zh.Big5]
That took 4 ms.
zh-classical.UTF-8.txt
TextCat 2.2 (out of place)
Result == [zh.UTF-8]
That took 4 ms.
zh.GB18030.txt
TextCat 2.2 (out of place)
Result == [zh.GB18030]
That took 4 ms.
zh.GB2312.txt
TextCat 2.2 (out of place)
Result == [zh.GB2312]
That took 4 ms.
zh-min-nan.UTF-8.txt
TextCat 2.2 (out of place)
Result == [hak.UTF-8]
That took 6 ms.
zh.UTF-8.txt
TextCat 2.2 (out of place)
Result == [zh.UTF-8]
That took 4 ms.
zh-yue.UTF-8.txt
TextCat 2.2 (out of place)
Result == [zh.UTF-8]
That took 4 ms.
$ 

$ cvs -d :pserver:username@cvs.dev.java.net:/cvs login
$ cvs -d :pserver:username@cvs.dev.java.net:/cvs checkout sampo
$ cd sampo/language
$ export JAVA_HOME=/usr/java/jdk1.6.0_04
$ export SAMPO_HOME=/var/sampo
$ export PATH=$SAMPO_HOME/bin:$JAVA_HOME/bin:$PATH
$ ant -DSAMPO_HOME=$SAMPO_HOME install
$ wikipedia-mainpage-html.sh
$ wikipedia-mainpage-text.sh
$ 

# yum install libtextcat
$ rpm -q libtextcat
libtextcat-2.2-4.fc8
libtextcat-2.2-4.fc8
$ rpm -ql libtextcat
/usr/lib64/libtextcat.so.0
/usr/lib64/libtextcat.so.0.0.0
/usr/share/doc/libtextcat-2.2
/usr/share/doc/libtextcat-2.2/ChangeLog
/usr/share/doc/libtextcat-2.2/LICENSE
/usr/share/doc/libtextcat-2.2/README
/usr/share/doc/libtextcat-2.2/TODO
/usr/share/libtextcat
/usr/share/libtextcat/afrikaans.lm
...
/usr/share/libtextcat/finnish.lm
/usr/share/libtextcat/fpdb.conf
/usr/share/libtextcat/french.lm
...
$ cat /usr/share/libtextcat/fpdb.conf
...
italian.lm                           it--utf8
japanese.lm                          ja--utf8
korean.lm                            ko--utf8
...
$ 

$ wget http://software.wise-guys.nl/download/libtextcat-2.2.tar.gz
$ tar zxvpf libtextcat-2.2.tar.gz
$ cd libtextcat-2.2
$ ./configure --prefix=/usr
$ make
$ cd src
$ cp testtextcat createfp $SAMPO_HOME/bin
$ cd /usr/share/libtextcat
$ cat $SAMPO_HOME/wikipedia-mainpage-text/en.UTF-8.txt | testtextcat fpdb.conf
TextCat 2.2 (out of place)
Result == [en--utf8]
That took 2 ms
$ cat $SAMPO_HOME/wikipedia-mainpage-text/ja.UTF-8.txt | testtextcat fpdb.conf
TextCat 2.2 (out of place)
Result == [mr--utf8][am--utf8][yi--utf8]
That took 2 ms.
$ 

$ wikipedia-mainpage-createfp.sh
$ cd $SAMPO_HOME/wikipedia-mainpage-text
$ testtextcat.sh *.txt
aa.UTF-8.txt
TextCat 2.2 (out of place)
Result == [aa.UTF-8]
That took 3 ms.
ab.UTF-8.txt
TextCat 2.2 (out of place)
Result == [ab.UTF-8]
That took 3 ms.
af.UTF-8.txt
TextCat 2.2 (out of place)
Result == [af.UTF-8]
That took 4 ms.
...
ja.UTF-8.txt
TextCat 2.2 (out of place)
Result == [ja.UTF-8]
That took 3 ms.
...
zu.UTF-8.txt
TextCat 2.2 (out of place)
Result == [zu.UTF-8]
That took 2 ms.
$ 

package net.java.sampo.usgov.fips;

import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;

public class State implements Serializable, Comparable<State> {
  private int seq;
  private String fipsCode;
  private String alphaCode;
  private String name;

  private State(int seq, String fipsCode, String alphaCode, String name) {
    this.seq = seq;
    this.fipsCode = fipsCode;
    this.alphaCode = alphaCode;
    this.name = name;
  }

  public int getSequenceNumber() {
    return seq;
  }

  public String getFipsCode() {
    return fipsCode;
  }

  public String getAlphaCode() {
    return alphaCode;
  }

  public String getName() {
    return name;
  }

  public int compareTo(State other) {
    return seq - other.seq;
  }

  // http://www.itl.nist.gov/fipspubs/fip55-3.htm
  public static final State AL = new State( 1, "01", "AL", "Alabama");
  public static final State AK = new State( 2, "02", "AK", "Alaska");
  public static final State AZ = new State( 3, "04", "AZ", "Arizona");
  public static final State AR = new State( 4, "05", "AR", "Arkansas");
  public static final State CA = new State( 5, "06", "CA", "California");
  public static final State CO = new State( 6, "08", "CO", "Colorado");
  public static final State CT = new State( 7, "09", "CT", "Connecticut"); // ?
  public static final State DE = new State( 8, "10", "DE", "Delaware");
  public static final State DC = new State( 9, "11", "DC", "District of Columbia");
  public static final State FL = new State(10, "12", "FL", "Florida");
  public static final State GA = new State(11, "13", "GA", "Georgia");
  public static final State HI = new State(12, "15", "HI", "Hawaii");
  public static final State ID = new State(13, "16", "ID", "Idaho");
  public static final State IL = new State(14, "17", "IL", "Illinois");
  public static final State IN = new State(15, "18", "IN", "Indiana");
  public static final State IA = new State(16, "19", "IA", "Iowa");
  public static final State KS = new State(17, "20", "KS", "Kansas");
  public static final State KY = new State(18, "21", "KY", "Kentucky");
  public static final State LA = new State(19, "22", "LA", "Louisiana");
  public static final State ME = new State(20, "23", "ME", "Maine");
  public static final State MD = new State(21, "24", "MD", "Maryland");
  public static final State MA = new State(22, "25", "MA", "Massachusetts");
  public static final State MI = new State(23, "26", "MI", "Michigan");
  public static final State MN = new State(24, "27", "MN", "Minnesota");
  public static final State MS = new State(25, "28", "MS", "Mississippi");
  public static final State MO = new State(26, "29", "MO", "Missouri");
  public static final State MT = new State(27, "30", "MT", "Montana");
  public static final State NE = new State(28, "31", "NE", "Nebraska");
  public static final State NV = new State(29, "32", "NV", "Nevada");
  public static final State NH = new State(30, "33", "NH", "New Hampshire");
  public static final State NJ = new State(31, "34", "NJ", "New Jersey");
  public static final State NM = new State(32, "35", "NM", "New Mexico");
  public static final State NY = new State(33, "36", "NY", "New York");
  public static final State NC = new State(34, "37", "NC", "North Carolina");
  public static final State ND = new State(35, "38", "ND", "North Dakota");
  public static final State OH = new State(36, "39", "OH", "Ohio");
  public static final State OK = new State(37, "40", "OK", "Oklahoma");
  public static final State OR = new State(38, "41", "OR", "Oregon");
  public static final State PA = new State(39, "42", "PA", "Pennsylvania");
  public static final State RI = new State(40, "44", "RI", "Rhode Island");
  public static final State SC = new State(41, "45", "SC", "South Carolina");
  public static final State SD = new State(42, "46", "SD", "South Dakota");
  public static final State TN = new State(43, "47", "TN", "Tennessee");
  public static final State TX = new State(44, "48", "TX", "Texas");
  public static final State UT = new State(45, "49", "UT", "Utah");
  public static final State VT = new State(46, "50", "VT", "Vermont");
  public static final State VA = new State(47, "51", "VA", "Virginia");
  public static final State WA = new State(48, "53", "WA", "Washington");
  public static final State WV = new State(49, "54", "WV", "West Virginia");
  public static final State WI = new State(50, "55", "WI", "Wisconsin");
  public static final State WY = new State(51, "56", "WY", "Wyoming");
  //
  public static final State AS = new State(52, "60", "AS", "American Samoa");
  public static final State GU = new State(54, "66", "GU", "Guam");
  public static final State MP = new State(56, "69", "MP", "Northern Mariana Islands");
  public static final State PR = new State(58, "72", "PR", "Puerto Rico");
  public static final State UM = new State(59, "74", "UM", "U.S. Minor Outlying Islands");
  public static final State VI = new State(60, "78", "VI", "Virgin Islands of the U.S.");
  //
  public static final State FM = new State(53, "64", "FM", "Federated States of Micronesia");
  public static final State MH = new State(55, "68", "MH", "Marshall Islands");
  public static final State PW = new State(57, "70", "PW", "Palau");

  private static final State[] allStates = new State[] {
    AL, AK, AZ, AR, CA, CO, CT, DE, DC, FL,
    GA, HI, ID, IL, IN, IA, KS, KY, LA, ME,
    MD, MA, MI, MN, MS, MO, MT, NE, NV, NH,
    NJ, NM, NY, NC, ND, OH, OK, OR, PA, RI,
    SC, SD, TN, TX, UT, VT, VA, WA, WV, WI,
    WY, AS, FM, GU, MH, MP, PW, PR, UM, VI
  };

  public static State[] getAllStates() {
    return allStates;
  }

  private static State[] fipsStates = new State[79];
  private static Map<String,State> alphaMap = new HashMap<String,State>();

  static {
    for (State state : allStates) {
      fipsStates[getIntFipsCode(state.getFipsCode())] = state;
      alphaMap.put(state.getAlphaCode(), state);
    }
  }

  private static int getIntFipsCode(String fipsCode) {
    if (fipsCode == null || fipsCode.length() != 2) {
      return -1;
    }
    int m = fipsCode.charAt(0) - '0';
    if (m < 0 || m > 9) {
      return -1;
    }
    int n = fipsCode.charAt(1) - '0';
    if (n < 0 || n > 9) {
      return -1;
    }
    return m * 10 + n;
  }

  public static State getStateByFipsCode(String fipsCode) {
    int n = getIntFipsCode(fipsCode);
    if (n < 1 || n >= fipsStates.length) {
      return null;
    }
    return fipsStates[n];
  }

  public static State getStateByAlphaCode(String alphaCode) {
    return alphaMap.get(alphaCode);
  }
} 

package net.java.sampo.immigration.processingTime.db;

import javax.sql.DataSource;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CreateTableCommand implements Command {
  private static Log log = LogFactory.getLog(CreateTableCommand.class);

  private static final String sql
    = "create table PROCESSINGTIME ("
    + "FACILITY varchar(32),"
    + "POSTEDDATE timestamp,"
    + "LINENO integer,"
    + "FORM varchar(8),"
    + "TITLE varchar(128),"
    + "CLASSIFICATION varchar(128),"
    + "TIMEFRAME varchar(32),"
    + "PROCESSINGDATE timestamp,"
    + "DAYS integer,"
    + "primary key(FACILITY, POSTEDDATE, LINENO))";

  private JdbcTemplate jdbcTemplate;

  public CreateTableCommand() {
  }

  public void setDataSource(DataSource dataSource) {
    jdbcTemplate = new JdbcTemplate(dataSource);
  }

  public void execute() {
    execute(new ContextBase());
  }

  public boolean execute(Context context) {
    try {
      jdbcTemplate.execute(sql);
    }
    catch (DataAccessException e) {
      log.error(e);
      return true;
    }
    return false;
  }

  public static void main(String[] args) throws Exception {
    Resource resource = new ClassPathResource("processingTime.xml");
    XmlBeanFactory factory = new XmlBeanFactory(resource);
    log.info("getting command bean ..");
    CreateTableCommand command
      = (CreateTableCommand)factory.getBean("createTableCommand");
    log.info("executing ...");
    command.execute();
  }
} 

<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="
      http://www.springframework.org/schema/beans
      http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
      http://www.springframework.org/schema/context
      http://www.springframework.org/schema/context/spring-context-2.5.xsd
    ">

  <bean id="createTableCommand"
     class="net.java.sampo.immigration.processingTime.db.CreateTableCommand">
    <property name="dataSource" ref="dataSource" />
  </bean>

  <bean id="dataSource" destroy-method="close"
     class="org.apache.commons.dbcp.BasicDataSource">
    <property name="driverClassName"
             value="org.apache.derby.jdbc.EmbeddedDriver" />
    <property name="url"
             value="jdbc:derby:processingTime.db;create=true" />
  </bean>
</beans>

package net.java.sampo.immigration.processingTime;

import java.io.Serializable;
import java.util.Date;
import java.util.Formatter;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static net.java.sampo.immigration.processingTime.ParserUtils.parseDate;

public class ProcessingTime implements Serializable {
  private static final Log log = LogFactory.getLog(ProcessingTime.class);

  private String facility;
  private Date date;
  private int lineNo;
  private String form;
  private String title; // name
  private String classification; // basis
  private String timeframe;

  public ProcessingTime(String facility, Date date, int lineNo,
           String form, String name, String timeframe) {
    this(facility, date, lineNo, form, name, null, timeframe);
  }

  public ProcessingTime(String facility, Date date, int lineNo,
           String form, String title, String classification, String timeframe) {
    this.facility = facility;
    this.date = date;
    this.lineNo = lineNo;
    this.form = form;
    this.title = title;
    this.classification = classification;
    this.timeframe = timeframe;
  }

  public String getFacility() {
    return facility;
  }

  public Date getDate() {
    return date;
  }

  public int getLineNO() {
    return lineNo;
  }

  public String getForm() {
    return form;
  }

  public String getTitle() {
    return title;
  }

  public String getName() {
    return title;
  }

  public String getClassification() {
    return classification;
  }

  public String getTimeframe() {
    return timeframe;
  }

  public String toString() {
    return toString(false);
  }

  public String toString(boolean additionalInfo) {
    Formatter f = new Formatter();
    f.format("[%s|%tF|%d|%s|%s|", facility, date, lineNo, form, title);
    if (classification != null) {
      f.format("%s|", classification);
    }
    f.format("%s", timeframe);
    if (additionalInfo) {
      f.format("|%tF|%d", getProcessingDate(), getTimeframeInDay());
    }
    f.format("]");
    return f.toString();
  }

  public Date getProcessingDate() {
    Date postedDate = date;
    if (timeframe == null || timeframe.length() == 0) {
      return null;
    }
    String s = timeframe.toLowerCase();
    char c = s.charAt(0);
    if (c >= 'a' && c <= 'z') {
      return parseDate(s);
    }
    int i = s.indexOf(' ');
    int n = 0 - Integer.parseInt(s.substring(0,i));
    s = s.substring(i+1);
    if (s.startsWith("day")) {
      return DateUtils.addDays(postedDate, n);
    }
    else if (s.startsWith("week")) {
      return DateUtils.addWeeks(postedDate, n);
    }
    else if (s.startsWith("month")) {
      return DateUtils.addMonths(postedDate, n);
    }
    else if (s.startsWith("year")) {
      return DateUtils.addYears(postedDate, n);
    }
    else if (s.startsWith("semana")) {
      return DateUtils.addWeeks(postedDate, n);
    }
    else if (s.startsWith("mese")) {
      return DateUtils.addMonths(postedDate, n);
    }
    else {
      log.warn("invalid time frame \"" + timeframe + "\"");
      return null;
    }
  }

  public int getTimeframeInDay() {
    Date postedDate = date;
    Date processingDate = getProcessingDate();
    if (processingDate == null) {
      return -1;
    }
    return (int)((postedDate.getTime() - processingDate.getTime()) / 86400000L);
    // 86400000 = 24 x 60 x 60 x 1000
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class ParserUtils {
  private static Log log = LogFactory.getLog(ParserUtils.class);

  public static String toTrimedString(Reader in) throws IOException {
    StringBuilder sb = new StringBuilder();
    for (String line : (List<String>)IOUtils.readLines(in)) {
      sb.append(line.trim());
    }
    return sb.toString();
  }

  public static String cutContent(byte[] bytes) throws IOException {
    Reader in = new InputStreamReader(new ByteArrayInputStream(bytes), "US-ASCII");
    return cutContent( toTrimedString(in) );
  }

  public static String cutContent(File file) throws IOException {
    Reader in = new InputStreamReader(new FileInputStream(file), "US-ASCII");
    return cutContent(in);
  }

  public static String cutContent(Reader in) throws IOException {
    return cutContent( toTrimedString(in) );
  }

  public static String cutContent(String html) {
    String sl = html.toLowerCase();
    int n = sl.indexOf("posted ");
    n = sl.indexOf("posted ", n+7);

    int m1 = sl.lastIndexOf("<b><big>", n);
    if (m1 < 0) { // NBC 2003-11-14
      m1 = n;
    }
    m1 = sl.lastIndexOf(">", m1);
    int m2 = sl.indexOf("<", n);
    StringBuilder sb = new StringBuilder();
    sb.append("<p align=\"center\">");
    sb.append(html.substring(m1+1,m2));
    sb.append("</p>");

    m1 = sl.indexOf("<table", m2);
    m2 = sl.indexOf("</table>", m1);
    sb.append(html.substring(m1,m2+8));
    return sb.toString();
  }

  public static String parseFacilityName(String content) {
    String s = content.toLowerCase();
    int n = s.indexOf("<big>");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"<big> \" tag not found.");
      }
      return "National Benefits Center"; // NBC 2003-11-14
    }
    int m = s.indexOf("</", n+5); // 2003-11-14
    if (m < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"</big> \" tag not found.");
      }
      return null;
    }
    String name = content.substring(n+5, m).trim();
    // 2003-11-14
    if (s.substring(0,n).indexOf("service center") >= 0
     && name.toLowerCase().indexOf("service center") < 0) {
      name = name + " Service Center";
    }
    return name;
  }

  public static Date parsePostedDate(String content) {
    int n = content.toLowerCase().indexOf("posted ");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"Posted \" not found.");
      }
      return null;
    }
    return parseDate(content.substring(n+7));
  }

  public static ProcessingTime[] parseProcessingTimes(String content) {
    String facility = parseFacilityName(content);
    Date date = parsePostedDate(content);
    int lineNo = 0;
    //
    String s = content.toLowerCase();
    int n = s.indexOf("form");
    List<ProcessingTime> list = new ArrayList<ProcessingTime>();
    while (true) {
      n = s.indexOf("<tr", n);
      if (n < 0) {
        break;
      }
      int m = s.indexOf("<tr", n+4); // CSC 2003-11-14 </tr> missing !!
      String line = null;
      if (m < 0) {
        line = content.substring(n);
      }
      else {
        line = content.substring(n, m);
      }
      list.add( parseLine(facility, date, lineNo, line) );
      lineNo ++;
      if (m < 0) {
        break;
      }
      n = m;
    }
    ProcessingTime[] array = new ProcessingTime[ list.size() ];
    list.toArray(array);
    return array;
  }

  private static ProcessingTime parseLine(String facility, Date date, int lineNo, String s) {
    if (log.isDebugEnabled()) {
      log.debug(s);
    }
    String sl = s.toLowerCase();
    int m = sl.indexOf("<b>");
    int n = sl.indexOf("<", m+3); // 2003-11-14
    List<String> list = new ArrayList<String>(4);
    list.add(s.substring(m+3, n).trim()); // form
    if (log.isDebugEnabled()) {
      log.debug(list);
    }
    while (true) {
      n = sl.indexOf("</td", n+5); // 2005-10-19 District Office
      if (n < 0) {
        break;
      }
      m = sl.lastIndexOf(">", n);
      list.add( s.substring(m+1, n).trim() );
      if (log.isDebugEnabled()) {
        log.debug(list);
      }
    }
    ProcessingTime processingTime = null;
    if (list.size() == 3) {
      processingTime = new ProcessingTime(facility, date, lineNo,
                             list.get(0), list.get(1), list.get(2));
    }
    else if (list.size() >= 4) {
      processingTime = new ProcessingTime(facility, date, lineNo,
                             list.get(0), list.get(1), list.get(2), list.get(3));
    }
    else {
      if (log.isWarnEnabled()) {
        log.warn("invalid field number = " + list.size() + " for " + s);
      }
    }
    return processingTime;
  }

  public static Date parseDate(String s) {
    int n = s.indexOf(' ');
    int m = s.indexOf(',', n+1);
    int dd = Integer.parseInt(s.substring(n+1,m));
    int yyyy = Integer.parseInt(s.substring(m+2,m+6));
    int mm = parseMonth(s.substring(0,n));
    if (mm < 1 || mm > 12) {
      return null;
    }
    Calendar cal = Calendar.getInstance();
    cal.set(yyyy, mm-1, dd, 12, 0, 0);
    return cal.getTime();
  }

  private static final String[] months = {
    "jan", "feb", "mar", "apr", "may", "jun",
    "jul", "aug", "sep", "oct", "nov", "dec"
  };

  private static int parseMonth(String s) {
    s = s.toLowerCase();
    for (int i = 0; i < months.length; i++) {
      if (s.startsWith(months[i])) {
        return i+1;
      }
    }
    return -1;
  }
} 

<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="
      http://www.springframework.org/schema/beans
      http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
      http://www.springframework.org/schema/context
      http://www.springframework.org/schema/context/spring-context-2.5.xsd
    ">

  <bean id="chain" class="net.java.sampo.immigration.processingTime.command.spring.CommandChain">
    <property name="commands">
      <list>
        <ref bean="fetchCommand" />
        <ref bean="cutContentCommand" />
        <ref bean="parseContentCommand" />
        <ref bean="setFilenameCommand" />
        <ref bean="writeBytesCommand" />
      </list>
    </property>
    <property name="contextMap">
      <map>
        <entry key="url"
               value="https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70" />
        <entry key="outPath" value="/tmp/pt" />
      </map>
    </property>
  </bean>

  <bean id="fetchCommand"
        class="net.java.sampo.immigration.processingTime.command.FetchCommand" />
  <bean id="cutContentCommand"
        class="net.java.sampo.immigration.processingTime.command.CutContentCommand" />
  <bean id="parseContentCommand"
        class="net.java.sampo.immigration.processingTime.command.ParseContentCommand" />
  <bean id="setFilenameCommand"
        class="net.java.sampo.immigration.processingTime.command.SetFilenameCommand" />
  <bean id="writeBytesCommand"
        class="net.java.sampo.immigration.processingTime.command.WriteBytesCommand" />
</beans>

package net.java.sampo.immigration.processingTime.command.spring;

import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CommandChain extends ChainBase implements InitializingBean {
  private static Log log = LogFactory.getLog(CommandChain.class);

  private List<Command> commands;
  private Map<String,String> contextMap;
  private Context context;

  public CommandChain() {
    context = null;
  }

  public void setCommands(List<Command> commands) {
    this.commands = commands;
  }

  public List<Command> getCommands() {
    return commands;
  }

  public void setContextMap(Map<String,String> contextMap) {
    this.contextMap = contextMap;
  }

  public Map<String,String> getContextMap() {
    return contextMap;
  }

  public Context getContext() {
    return context;
  }

  public void afterPropertiesSet() {
    // adding commands
    for (Command command : commands) {
      addCommand(command);
    }
    // setting context
    context = new ContextBase();
    for (String key : contextMap.keySet()) {
      String value = contextMap.get(key);
      context.put(key, value);
    }
  }

  public void execute() throws Exception {
    super.execute(context);
  }

  public static void main(String[] args) throws Exception {
    Resource resource = new ClassPathResource("chain.xml");
    XmlBeanFactory factory = new XmlBeanFactory(resource);
    log.info("getting chain ..");
    CommandChain chain = (CommandChain)factory.getBean("chain");
    Context context = chain.getContext();
    log.info("context=" + context);
    log.info("executing ...");
    chain.execute();
    //
    context.remove("bytes");
    context.remove("content");
    context.remove("processingTimes");
    log.info("context=" + context);
  }
} 

$ cat log4j.properties
log4j.rootCategory=WARN,A1
log4j.category.net.java.sampo=INFO
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c{1} - %m%n
$ java -classpath .:lib/commons-logging-1.1.1.jar:lib/log4j-1.2.15.jar:lib/commons-chain-1.1.jar\
:lib/commons-io-1.3.2.jar:lib/spring-2.5.jar \
net.java.sampo.immigration.processingTime.command.spring.CommandChain
2007-12-16 07:39:34,508 [main] INFO  CommandChain - getting chain ..
2007-12-16 07:39:34,569 [main] INFO  CommandChain \
- context={outPath=/tmp/pt, url=https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70}
2007-12-16 07:39:34,569 [main] INFO  CommandChain - executing ...
2007-12-16 07:39:36,012 [main] INFO  CommandChain \
- context={outPath=/tmp/pt, facility=San Jose CA, posted=Wed Nov 14 12:00:00 PST 2007,\
 url=https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=70,\
 outFile=/tmp/pt/San_Jose_CA-20071114.html}
$ 

package net.java.sampo.immigration.processingTime.command.spring;

import org.springframework.beans.factory.xml.XmlBeanFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;

public class CommandExecutor {
  public static void main(String[] args) throws Exception {
    Resource resource = new ClassPathResource("command.xml");
    XmlBeanFactory factory = new XmlBeanFactory(resource);
    Command command = (Command)factory.getBean("command");
    Context context = (Context)factory.getBean("context");
    command.execute(context);
  }
} 

package net.java.sampo.immigration.processingTime.command.spring;

import java.util.HashMap;
import org.apache.commons.chain.Context;

public class CommandContext extends HashMap implements Context {
  public CommandContext() {
    super();
  }

  public String getOutPath() {
    return (String)get("outPath");
  }

  public void setOutPath(String outPath) {
    put("outPath", outPath);
  }
}

<?xml version="1.0" encoding="utf-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="
      http://www.springframework.org/schema/beans
      http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
      http://www.springframework.org/schema/context
      http://www.springframework.org/schema/context/spring-context-2.5.xsd
    ">

  <bean id="command" class="net.java.sampo.immigration.processingTime.command.FetchAllCommand" />
  <bean id="context" class="net.java.sampo.immigration.processingTime.command.spring.CommandContext">
    <property name="outPath" value="/tmp/pt" />
  </bean>
</beans>

    Chain command = new ChainBase();
    command.addCommand(new FetchCommand());
    command.addCommand(new CutContentCommand());
    command.addCommand(new ParseContentCommand());
    command.addCommand(new SetFilenameCommand());
    command.addCommand(new WriteBytesCommand());

    for (String facility : urlMap.getAllFacilityNames()) {
      String url = urlMap.getUrl(facility);
      Context eachContext = new ContextBase();
      eachContext.put("url", url);
      eachContext.put("outPath", outPath);
      command.execute(eachContext);
    } 

public class FetchCommand implements Command {
  private static Log log = LogFactory.getLog(FetchCommand.class);

  public boolean execute(Context context) {
    String url = (String)context.get("url");
    if (url == null) {
      log.error("url not set in context.");
      return true;
    }
    byte[] bytes = null;
    try {
      bytes = EgovFetcher.getInstance().fetch(url);
    }
    catch (IOException e) {
      log.error(e);
      return true;
    }
    if (bytes == null) {
      return true;
    }
    context.put("bytes", bytes);
    return false;
  }
} 

public class CutContentCommand implements Command {
  private static Log log = LogFactory.getLog(CutContentCommand.class);

  public boolean execute(Context context) {
    byte[] bytes = (byte[])context.get("bytes");
    if (bytes == null) {
      log.error("bytes not set in context.");
      return true;
    }
    String content = null;
    try {
      content = ParserUtils.cutContent(bytes);
    }
    catch (Exception e) {
      log.error(e);
    }
    if (content == null) {
      return true;
    }
    context.put("content", content);
    return false;
  }
} 

public class ParseContentCommand implements Command {
  private static Log log = LogFactory.getLog(ParseContentCommand.class);

  public boolean execute(Context context) {
    String content = (String)context.get("content");
    if (content == null) {
      log.error("content not set in context.");
      return true;
    }
    String facility = ParserUtils.parseFacilityName(content);
    context.put("facility", facility);
    Date date = ParserUtils.parsePostedDate(content);
    context.put("posted", date);
    ProcessingTime[] times = ParserUtils.parseProcessingTimes(content);
    context.put("processingTimes", times);
    return false;
  }
} 

public class SetFilenameCommand implements Command {
  private static Log log = LogFactory.getLog(SetFilenameCommand.class);

  public boolean execute(Context context) {
    String path = (String)context.get("outPath");
    if (path == null) {
      log.error("outPath not set in context.");
    }
    String facility = (String)context.get("facility");
    Date date = (Date)context.get("posted");
    if (facility == null || date == null) {
      if (new ParseContentCommand().execute(context)) {
        log.error("facility, posted, or context not set in context.");
        return true;
      }
      facility = (String)context.get("facility");
      date = (Date)context.get("posted");
    }
    facility = facility.replaceAll("[\\s]+", "_");
    Formatter f = new Formatter();
    f.format("%s/%s-%tY%tm%td.html", path, facility, date, date, date);
    String filename = f.toString();
    context.put("outFile", filename);
    return false;
  }
} 

public class WriteBytesCommand implements Command {
  private static Log log = LogFactory.getLog(WriteBytesCommand.class);

  public boolean execute(Context context) {
    byte[] bytes = (byte[])context.get("bytes");
    if (bytes == null) {
      log.error("bytes not set in context.");
      return true;
    }
    String filename = (String)context.get("outFile");
    if (filename == null) {
      log.error("outFile not set in context.");
      return true;
    }
    try {
      FileUtils.writeByteArrayToFile(new File(filename), bytes);
    }
    catch (IOException e) {
      log.error(e);
    }
    return false;
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.InputStream;
import java.io.IOException;
import java.net.URL;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class EgovFetcher {
  private static final long minimumFetchInterval = 1000; // 1 sec

  private static Log log = LogFactory.getLog(EgovFetcher.class);

  private static EgovFetcher fetcher = null;

  public static EgovFetcher getInstance() {
    if (fetcher == null) {
      fetcher = new EgovFetcher();
    }
    return fetcher;
  }

  private long lastFetchedTime;

  public EgovFetcher() {
    lastFetchedTime = -1L;
  }

  synchronized public byte[] fetch(String url) throws IOException {
    if (url == null) {
      log.error("url is null.");
      return null;
    }
    long now = System.currentTimeMillis();
    long t = minimumFetchInterval - (now - lastFetchedTime);
    if (t > 0L) {
      if (log.isInfoEnabled()) {
        log.info("sleeping " + t + "msecs ..");
      }
      try {
        Thread.sleep(t);
      }
      catch (InterruptedException e) {
      }
      now = System.currentTimeMillis();
    }
    lastFetchedTime = now;
    byte[] bytes = null;
    InputStream in = new URL(url).openStream();
    bytes = IOUtils.toByteArray(in);
    return bytes;
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class FacilityUrlMap extends HashMap<String, String> {
  private static final String ptimesUrl = " https://egov.uscis.gov/cris/jsps/ptimes.jsp";
  private static final String officeUrl
    = "https://egov.uscis.gov/cris/jsps/officeProcesstimes.jsp?selectedOffice=";
  private static final String centerUrl
    = "https://egov.uscis.gov/cris/jsps/Processtimes.jsp?SeviceCenter=";
  private static final String nbcUrl = "https://egov.uscis.gov/cris/jsps/NBCprocesstimes.jsp";

  private static Log log = LogFactory.getLog(FacilityUrlMap.class);

  private static FacilityUrlMap map;

  public static FacilityUrlMap getInstance() {
    if (map == null) {
      map = new FacilityUrlMap();
    }
    return map;
  }

  private String[] facilityNames;

  public FacilityUrlMap() {
    super();
    try {
      init();
    }
    catch (IOException e) {
      log.fatal(e);
    }
  }

  public String getUrl(String facility) {
    return get(facility);
  }

  public String[] getAllFacilityNames() {
    return facilityNames;
  }

  private void init() throws IOException {
    byte[] bytes = EgovFetcher.getInstance().fetch(ptimesUrl);   
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new ByteArrayInputStream(bytes), "ISO-8859-1"));
    parse(in);
  }

  private void parse(BufferedReader in) throws IOException {
      List<String> list = new ArrayList<String>();
      int mode = 0;
      while (true) {
        String line = in.readLine();
        if (log.isDebugEnabled()) {
          log.debug("" + mode + " : " + line);
        }
        if (line == null) {
          break;
        }
        String s = line.toLowerCase();
        if (mode == 0) {
          if (s.indexOf("<select name=\"selectedoffice\">") >= 0) {
            mode = 1;
          }
          else if (s.indexOf("<select name=\"sevicecenter\">") >= 0) { // Sevice : "r" missing !!
            mode = 2;
          }
          continue;
        }
        if (s.indexOf("</select>") >= 0) {
          if (mode == 2) {
            break;
          }
          mode = 0;
          continue;
        }
        //
        int n = 0;
        while (true) {
          int m = s.indexOf("<option value=\"", n);
          if (m < 0) {
            break;
          }
          n = s.indexOf('"', m+15);
          String id = line.substring(m+15,n).trim();
          m = s.indexOf('<', n+2);
          String name = line.substring(n+2, m).trim();
          log.info(id + " : " + name);
          if (mode == 1) {
            int intId = Integer.parseInt(id);
            put(name, officeUrl + intId);
            list.add(name);
          }
          else if (mode == 2) {
            put(name, centerUrl + id);
            list.add(name);
          }
          n = m+8;
        }
      }
      String name = "National Benefits Center";
      put(name, nbcUrl);
      list.add(name);
      //
      int n = list.size();
      facilityNames = new String[n];
      list.toArray(facilityNames);
  }
} 

package net.java.sampo.immigration.processingTime.command;

import java.io.File;
import java.io.IOException;
import java.util.Map;
import org.apache.commons.chain.Chain;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import net.java.sampo.immigration.processingTime.FacilityUrlMap;

public class FetchAllCommand implements Command {
  private static Log log = LogFactory.getLog(FetchAllCommand.class);

  public boolean execute(Context context) {
    FacilityUrlMap urlMap = FacilityUrlMap.getInstance();
    if (urlMap == null) {
      log.fatal("facility to url map is null.");
      return true;
    }
    String outPath = (String)context.get("outPath");
    if (outPath == null) {
      log.error("outPath not set in context.");
      return true;
    }
    File outDir = new File(outPath);
    if (! outDir.exists()) {
      log.warn("outPath not exist. creating..");
      if (! outDir.mkdir()) {
        log.warn("outPath failed to create.");
        return true;
      }
    }
    Chain command = new ChainBase();
    command.addCommand(new FetchCommand());
    command.addCommand(new CutContentCommand());
    command.addCommand(new ParseContentCommand());
    command.addCommand(new SetFilenameCommand());
    command.addCommand(new WriteBytesCommand());

    for (String facility : urlMap.getAllFacilityNames()) {
      String url = urlMap.getUrl(facility);
      Context eachContext = new ContextBase();
      eachContext.put("url", url);
      eachContext.put("outPath", outPath);
      if (log.isInfoEnabled()) {
        log.info(facility + " : " + url + " to " + outPath);
      }
      try {
        command.execute(eachContext);
      }
      catch (Exception e) {
        log.error(e);
      }
    }
    return false;
  }

  public static void main(String[] args) throws Exception {
    String path = "orig";
    if (args.length > 0) {
      path = args[0];
    }
    Context context = new ContextBase();
    context.put("outPath", path);
    Chain command = new ChainBase();
    command.addCommand(new FetchAllCommand());
    command.execute(context);
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.File;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.sun.syndication.feed.synd.SyndContent;
import com.sun.syndication.feed.synd.SyndContentImpl;
import com.sun.syndication.feed.synd.SyndEntry;
import com.sun.syndication.feed.synd.SyndEntryImpl;
import com.sun.syndication.feed.synd.SyndFeedImpl;
import com.sun.syndication.feed.synd.SyndLink;
import com.sun.syndication.feed.synd.SyndLinkImpl;
import com.sun.syndication.feed.synd.SyndPerson;
import com.sun.syndication.feed.synd.SyndPersonImpl;
import com.sun.syndication.io.FeedException;
import com.sun.syndication.io.SyndFeedOutput;
import static net.java.sampo.immigration.processingTime.ProcessingTimesPageParser.parsePostedDate;

public class ProcessingTimeFeed extends SyndFeedImpl {
  private static final String urlPrefix = "http://localhost/immigration/";

  private static Log log = LogFactory.getLog(ProcessingTimeFeed.class);

  private ProcessingFacility facility;

  public ProcessingTimeFeed(ProcessingFacility facility) {
    this.facility = facility;
    setFeed();
  }

  private void setFeed() {
    String title = facility.getName() + " Processing Times";
    String url = urlPrefix + facility.getStringId() + ".xml";
    String urlAlt = urlPrefix + facility.getStringId() + ".html";
    //
    setFeedType("atom_1.0");
    setTitle(title);
    setDescription(title);
    setPublishedDate(new Date());
    //
    List<SyndLink> links = new ArrayList<SyndLink>();
    SyndLink link = new SyndLinkImpl();
    link.setHref(urlAlt);
    link.setRel("alternate");
    link.setType("text/html");
    links.add(link); // 0
    //
    link = new SyndLinkImpl();
    link.setHref(url);
    link.setRel("self");
    link.setType("text/xml");
    links.add(link); // 1
    //
    link = new SyndLinkImpl();
    link.setHref(url);
    link.setRel("service.feed");
    link.setType("application/x.atom+xml");
    link.setTitle(title);
    links.add(link); // 2
    setLinks(links);
    //
    List<SyndPerson> authors = new ArrayList<SyndPerson>();
    SyndPerson author = new SyndPersonImpl();
    author.setName("Kaz Muzik");
    authors.add(author);
    setAuthors(authors);
  }

  public void add(String contentText) {
    Date date = parsePostedDate(contentText);
    add(date, contentText);
  }

  public void add(Date date, String contentText) {
    Formatter f = new Formatter();
    f.format("%s%s-%tY%tm%td.html", urlPrefix, facility.getStringId(), date, date, date);
    String url = f.toString();
    f = new Formatter();
    f.format("%tF @ %s", date, facility.getName());
    String title = f.toString();
    //
    SyndEntry entry = new SyndEntryImpl();
    entry.setPublishedDate(date);
    entry.setTitle(title);
    //
    List<SyndContent> contents = new ArrayList<SyndContent>();
    SyndContent content = new SyndContentImpl();
    content.setValue(contentText);
    content.setType("html");
    contents.add(content);
    entry.setContents(contents);
    //
    List<SyndLink> links = new ArrayList<SyndLink>();
    SyndLink link = new SyndLinkImpl();
    link.setHref(url);
    link.setRel("alternate");
    link.setType("text/html");
    links.add(link);
    entry.setLinks(links);
    //
    List<SyndPerson> authors = new ArrayList<SyndPerson>();
    SyndPerson author = new SyndPersonImpl();
    author.setName("Kaz Muzik");
    authors.add(author);
    entry.setAuthors(authors);
    //
    List<SyndEntry> entryList = (List<SyndEntry>)getEntries();
    if (entryList == null) {
      entryList = new ArrayList<SyndEntry>();
      setEntries(entryList);
    }
    entryList.add(entry);
  }

  public String getXML() {
    try {
      return new SyndFeedOutput().outputString(this);
    }
    catch (FeedException e) {
      log.error(e);
      return null;
    }
  }

  public static void main(String[] args) throws Exception {
    ProcessingFacility facility = new FieldOffice(70, "San Jose CA");
    ProcessingTimeFeed feed = new ProcessingTimeFeed(facility);
    String content = FileUtils.readFileToString(new File("contents/070-20071114.html"));
    feed.add(content);
    System.out.println(feed.getXML());
  }
} 

<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/">
  <title>San Jose CA Processing Times</title>
  <link rel="alternate" href="http://localhost/immigration/070.html" />
  <link rel="self" href="http://localhost/immigration/070.xml" />
  <link rel="service.feed" href="http://localhost/immigration/070.xml" />
  <author>
    <name>Kaz Muzik</name>
  </author>
  <subtitle>San Jose CA Processing Times</subtitle>
  <updated>2007-12-09T03:07:41Z</updated>
  <dc:date>2007-12-09T03:07:41Z</dc:date>
  <entry>
    <title>2007-11-14 @ San Jose CA</title>
    <link rel="alternate" type="text/html"
href="http://localhost/immigration/070-20071114.html" />
    <author>
      <name>Kaz Muzik</name>
    </author>
    <updated>2007-11-14T20:00:00Z</updated>
    <published>2007-11-14T20:00:00Z</published>
    <content type="html">&lt;table border="0" cellpadding="0" cellspacing="0"
width="100%"&gt;&lt;tbody&gt;&lt;tr&gt;&lt;td align="center"&gt;District
Office Processing Dates for &lt;b&gt;&lt;big&gt;San Jose
CA&lt;/big&gt;&lt;/b&gt; Posted November 14,
2007&lt;/td&gt;&lt;/tr&gt;&lt;/tbody&gt;&lt;/table&gt;&lt;table
style="font-size: 10pt;" border="1" bordercolor="#045290" cellpadding="2"
cellspacing="0" width="100%"&gt;&lt;tbody&gt;&lt;tr
bgcolor="#000000"&gt;&lt;td width="45"&gt;&lt;font
color="#ffd200"&gt;Form&lt;/font&gt;&lt;/td&gt;&lt;td
align="center"&gt;&lt;font color="#ffd200"&gt;Form
Name&lt;/font&gt;&lt;/td&gt;&lt;td align="center"&gt;&lt;font
color="#ffd200"&gt;Processing
Timeframe:&lt;/font&gt;&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#ffff99"&gt;&lt;td&gt;&lt;b&gt;I-131&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application for Travel Documents&lt;/td&gt;&lt;td
align="center"&gt;3 Months&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#99ff99"&gt;&lt;td&gt;&lt;b&gt;I-485&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application to Register Permanent Residence or Adjust
Status&lt;/td&gt;&lt;td align="center"&gt;6 Months&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#ffff99"&gt;&lt;td&gt;&lt;b&gt;I-600&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Petition to Classify Orphan as an Immediate
Relative&lt;/td&gt;&lt;td align="center"&gt;July 05,
2007&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#99ff99"&gt;&lt;td&gt;&lt;b&gt;I-600A&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application for Advance Processing of Orphan
Petition&lt;/td&gt;&lt;td align="center"&gt;July 05,
2007&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#ffff99"&gt;&lt;td&gt;&lt;b&gt;I-765&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application for Employment Authorization&lt;/td&gt;&lt;td
align="center"&gt;11 Weeks&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#99ff99"&gt;&lt;td&gt;&lt;b&gt;N-400&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application for Naturalization&lt;/td&gt;&lt;td
align="center"&gt;March 08, 2007&lt;/td&gt;&lt;/tr&gt;&lt;tr
bgcolor="#ffff99"&gt;&lt;td&gt;&lt;b&gt;N-600&lt;/b&gt;&lt;/td&gt;&lt;td
align="left"&gt;Application for Certification of Citizenship&lt;/td&gt;&lt;td
align="center"&gt;October 12,
2006&lt;/td&gt;&lt;/tr&gt;&lt;/tbody&gt;&lt;/table&gt;</content>
    <dc:date>2007-11-14T20:00:00Z</dc:date>
  </entry>
</feed>　

The Case Status Online, Processing Times, Office Locator, and Change of Address Online Systems \
will be experiencing a scheduled outage from 9:30 PM EST
on Friday, December 7 to 8:00 AM EST on Monday, December 10 due to scheduled maintenance. 

package net.java.sampo.immigration.processingTime;

import java.io.Serializable;

public class ProcessingTime implements Serializable {
  private String form;
  private String title; // name
  private String classification; // basis
  private String timeframe;

  public ProcessingTime(String form, String name, String timeframe) {
    this(form, name, null, timeframe);
  }

  public ProcessingTime(String form, String title, String classification, String timeframe) {
    this.form = form;
    this.title = title;
    this.classification = classification;
    this.timeframe = timeframe;
  }

  public String getForm() {
    return form;
  }

  public String getTitle() {
    return title;
  }

  public String getName() {
    return title;
  }

  public String getClassification() {
    return title;
  }

  public String getTimeframe() {
    return timeframe;
  }

  public String toString() {
    return "PT[" + form + "|" + title + "|"
         + ((classification==null)?"":(classification+"|"))
         + timeframe + "]";
  }
} 

package net.java.sampo.immigration.processingTime;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import static net.java.sampo.immigration.processingTime.ProcessingTimeUtils.parseDate;

public class ProcessingTimesPageParser extends BufferedReader {
  private static Log log = LogFactory.getLog(ProcessingTimesPageParser.class);

  private String facilityName;
  private Date date;
  private ProcessingTime[] processingTimes;
  private String content;

  public ProcessingTimesPageParser(Reader in) {
    super(in);
  }

  public ProcessingTime[] getProcessingTimes() {
    return processingTimes;
  }

  public String getFacilityName() {
    return facilityName;
  }

  public Date getPostedDate() {
    return date;
  }

  public String getContent() {
    return content;
  }

  public void parseAll() throws IOException {
    content = parseContent();
    if (log.isDebugEnabled()) {
      log.debug(content);
    }
    facilityName = parseFacility(content);
    date = parsePostedDate(content);
    processingTimes = parseTable(content);
  }

  private String parseContent() throws IOException {
    int postedCount = 0;
    int endTableCount = 0;
    StringBuilder sb = new StringBuilder();
    String tableTag = null;
    while (true) {
      String line = readLine();
      if (line == null) {
        break;
      }
      line = line.trim();
      String lineLowerCase = line.toLowerCase();
      if (lineLowerCase.indexOf("<table") >= 0 && postedCount < 2) {
        tableTag = line;
      }
      if (lineLowerCase.indexOf("<tr") >= 0 && postedCount < 2) {
        sb = new StringBuilder();
        sb.append(tableTag);
        if (line.indexOf("<tbody") < 0) {
          sb.append("<tbody>");
        }
      }
      sb.append(line);
      if (lineLowerCase.indexOf("posted ") >= 0) {
        postedCount ++;
      }
      if (lineLowerCase.indexOf("</table>") >= 0 && postedCount >= 2) {
        endTableCount ++;
        if (endTableCount >= 2) {
          break;
        }
      }
    }
    return sb.toString();
  }

  private String parseFacility(String content) {
    String contentLowerCase = content.toLowerCase();
    int n = contentLowerCase.indexOf("<big>");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"<big> \" tag not found.");
      }
      return null;
    }
    int m = contentLowerCase.indexOf("</big>", n+5);
    if (m < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"</big> \" tag not found.");
      }
      return null;
    }
    return content.substring(n+5, m);
  }

  private Date parsePostedDate(String content) {
    int n = content.toLowerCase().indexOf("posted ");
    if (n < 0) {
      if (log.isWarnEnabled()) {
        log.warn("\"Posted \" not found.");
      }
      return null;
    }
    return parseDate(content.substring(n+7));
  }

  private ProcessingTime[] parseTable(String content) {
    String contentLowerCase = content.toLowerCase();
    int n = contentLowerCase.indexOf("processing timeframe");
    List<ProcessingTime> list = new ArrayList<ProcessingTime>();
    while (true) {
      n = contentLowerCase.indexOf("<tr", n);
      if (n < 0) {
        break;
      }
      int m = contentLowerCase.indexOf("</tr>", n+4);
      list.add( parseLine( content.substring(n, m+5) ) );
      n = m;
    }
    ProcessingTime[] array = new ProcessingTime[ list.size() ];
    list.toArray(array);
    return array;
  }

  private ProcessingTime parseLine(String s) {
    if (log.isDebugEnabled()) {
      log.debug(s);
    }
    String sl = s.toLowerCase();
    int m = sl.indexOf("<b>");
    int n = sl.indexOf("</b>", m+3);
    List<String> list = new ArrayList<String>(4);
    list.add(s.substring(m+3, n).trim()); // form
    if (log.isDebugEnabled()) {
      log.debug(list);
    }
    while (true) {
      n = sl.indexOf("</td>", n+5);
      if (n < 0) {
        break;
      }
      m = sl.lastIndexOf(">", n);
      list.add( s.substring(m+1, n).trim() );
      if (log.isDebugEnabled()) {
        log.debug(list);
      }
    }
    ProcessingTime processingTime = null;
    if (list.size() == 3) {
      processingTime = new ProcessingTime(list.get(0), list.get(1), list.get(2));
    }
    else if (list.size() == 4) {
      processingTime = new ProcessingTime(list.get(0), list.get(1), list.get(2), list.get(3));
    }
    else {
      if (log.isWarnEnabled()) {
        log.warn("invalid field number = " + list.size());
      }
    }
    return processingTime;
  }

  public static void main(String[] args) throws Exception {
    ProcessingTimesPageParser parser
      = new ProcessingTimesPageParser(new InputStreamReader(System.in, "ISO-8859-1"));
    parser.parseAll();
    System.out.printf("%s : %tF%n", parser.getFacilityName(), parser.getPostedDate());
    for (ProcessingTime pt : parser.getProcessingTimes()) {
      System.out.println(pt.toString());
    }
    System.out.println(parser.getContent());
  }
} 

package net.java.sampo.immigration.processingTime;

import java.util.Calendar;
import java.util.Date;
import java.util.Formatter;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class ProcessingTimeUtils {
  private static Log log = LogFactory.getLog(ProcessingTimeUtils.class);

  public static Date parseDate(String s) {
    int n = s.indexOf(' ');
    int m = s.indexOf(',', n+1);
    int dd = Integer.parseInt(s.substring(n+1,m));
    int yyyy = Integer.parseInt(s.substring(m+2,m+6));
    int mm = parseMonth(s.substring(0,n));
    if (mm < 1 || mm > 12) {
      return null;
    }
    Calendar cal = Calendar.getInstance();
    cal.set(yyyy, mm-1, dd);
    return cal.getTime();
  }

  private static final String[] months = {
    "jan", "feb", "mar", "apr", "may", "jun",
    "jul", "aug", "sep", "oct", "nov", "dec"
  };

  private static int parseMonth(String s) {
    s = s.toLowerCase();
    for (int i = 0; i < months.length; i++) {
      if (s.startsWith(months[i])) {
        return i+1;
      }
    }
    return -1;
  }
} 

package net.java.sampo.eijiro;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.chain.Chain;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class EijiroFormatChecker {

  private static Log log = LogFactory.getLog(EijiroFormatChecker.class);

  private Context context;
  private Command checkLineCommand;

  public EijiroFormatChecker() throws IOException {
    context = new ContextBase();
    context.put("errors", new ArrayList<String>());
    checkLineCommand = new CheckLineCommand();
  }

  public void close() throws IOException {
    context = null;
  }

  public List<String> getErrorLines() {
    return (List<String>)context.get("errors");
  }

  public void checkFile() throws IOException {
    checkFile(108);
  }

  public void checkFile(int ver) throws IOException {
    checkFile("EIJI-" + ver + ".TXT", "Shift-JIS");
  }

  public void checkFile(String filename, String encoding) throws IOException {
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new FileInputStream(filename), encoding));
    context.put("reader", in);
    context.put("count", 0);
    if (log.isInfoEnabled()) {
      log.info("Started checking \"" + filename + "\" ...");
    }

    while (true) {
      try {
        checkLineCommand.execute(context);
      }
      catch (Exception e) {
        log.error(e.getMessage(), e);
      }
      if (context.get("line") == null) {
        break;
      }
    }
    if (log.isInfoEnabled()) {
      log.info("" + context.get("count") + " lines processed, completed.");
    }
    if (log.isWarnEnabled()) {
      log.warn("" + getErrorLines().size() + " errors found, "
                  + context.get("count") + " lines read.");
    }
    in.close();
  }

  private static class CheckLineCommand extends ChainBase {
    public CheckLineCommand() {
      super();
      addCommand(new ReadLineCommand());
      addCommand(new CountUpCommand());
      addCommand(new SeparateLineCommand());
      addCommand(new CheckParenthesisChain());
    }
  }

  private static class CountUpCommand implements Command {
    public CountUpCommand() {
    }

    public boolean execute(Context context) {
      Integer countObj = (Integer)context.get("count");
      if (countObj == null) {
        log.error("counter is not set.");
        return true;
      }
      int count = countObj.intValue() + 1;
      context.put("count", new Integer(count));
      if (count % 100000 == 0) {
        if (log.isInfoEnabled()) {
          log.info("" + count + " lines processed.");
        }
      }
      return false;
    }
  }

  private static class ReadLineCommand implements Command {
    public ReadLineCommand() {
    }

    public boolean execute(Context context) throws IOException {
      BufferedReader in = (BufferedReader)context.get("reader");
      String line = in.readLine();
      context.put("line", line);
      return (line == null)?true:false;
    }
  }

  private static class SeparateLineCommand implements Command {
    public SeparateLineCommand() {
    }

    public boolean execute(Context context) {
      String line = (String)context.get("line");
      if (line == null) {
        log.error("line is null.");
        return true;
      }
      int len = line.length();
      if (len < 2) {
        log.warn("line length is less than 2.");
        return true;
      }
      List<String> fieldList = new ArrayList<String>();
      int n = 1;
      boolean ex = false;
      for (int i = 1; i < len; i++) {
        char c = line.charAt(i);
        boolean match = false;
        if (c == ':') {
          if (len > i + 2 && line.charAt(i-1) == ' ' && line.charAt(i+1) == ' ') {
            fieldList.add( line.substring(n,i-1) );
            n = i + 2;
            i++;
          }
          continue;
        }
        else if (c == '■') {
          if (len <= i + 2) {
            log.warn("invalid example mark in \"" + line + "\"");
            return true;
          }
          if (line.charAt(i+1) != '・') {
            log.warn("invalid example mark in \"" + line + "\"");
            return true;
          }
          fieldList.add( line.substring(n,i) );
          n = i + 2;
          i++;
          ex = true;
          continue;
        }
        else if (c == '◆') {
          fieldList.add( line.substring(n,i) );
          n = i + 1;
          ex = true;
          continue;
        }
        else if (c == '、' && (! ex)) {
          fieldList.add( line.substring(n,i) );
          n = i + 1;
          continue;
        }
      }
      fieldList.add( line.substring(n) );
      context.put("fields", fieldList);
      return false;
    }
  }

  private static class CheckParenthesisChain extends ChainBase {
    public CheckParenthesisChain() {
      super();
      addCommand(new CheckParenthesisCommand('{', '}'));  // 品詞ラベル
      addCommand(new CheckParenthesisCommand('\u3010', '\u3011'));
                                     // 【】 特殊フィールド, スピーチラベル(?)
      addCommand(new CheckParenthesisCommand('\uff5b', '\uff5d'));  // ｛｝ 読み仮名
      addCommand(new CheckParenthesisCommand('\u300a', '\u300b'));  // 《》 分野ラベル
      addCommand(new CheckParenthesisCommand('\u3008', '\u3009'));  // 〈〉 分野ラベル
      addCommand(new CheckParenthesisCommand('<', '>'));
      addCommand(new CheckParenthesisCommand('(', ')'));
      addCommand(new CheckParenthesisCommand('[', ']'));
    }
  }

  private static class CheckParenthesisCommand implements Command {
    private char startChar;
    private char endChar;

    public  CheckParenthesisCommand(char startChar, char endChar) {
      this.startChar = startChar;
      this.endChar = endChar;
    }

    public boolean execute(Context context) {
      String line = (String)context.get("line");
      if (line == null) {
        log.error("line is null.");
        return true;
      }
      List<String> fields = (List<String>)context.get("fields");
      if (fields == null || fields.size() < 1) {
        log.error("fields is null or empty in \"" + line + "\"");
        return true;
      }
      int level = 0;
      for (String s : fields) {
        int len = s.length();
        for (int i = 0; i < len; i++) {
          char c = s.charAt(i);
          if (c == startChar) {
            level ++;
          }
          else if (c == endChar) {
            if (level == 0) {
              ((List<String>)context.get("errors")).add(line);
              if (log.isWarnEnabled()) {
                log.warn("Unmatched \"" + endChar + "\" in \"" + s + "\"");
              }
              return true;
            }
            level --;
          }
        }
        if (level > 0) {
          ((List<String>)context.get("errors")).add(line);
          if (log.isWarnEnabled()) {
            log.warn("Unmatched \"" + startChar + "\" in \"" + s + "\"");
          }
          return true;
        }
      }
      return false;
    }
  }

  public static void main(String[] args) throws Exception {
    EijiroFormatChecker checker = new EijiroFormatChecker();
    if (args.length < 1) {
      checker.checkFile();
    }
    else if (args.length == 1) {
      int ver = Integer.parseInt(args[0]);
      checker.checkFile(ver);
    }
    else {
      checker.checkFile(args[0], args[1]);
    }
    System.out.println();
    for (String line : checker.getErrorLines()) {
      System.out.println(line);
    }
    checker.close();
  }
} 

$ cat check.sh
CP=lib/eijiro-lucene.jar:lib/commons-logging-1.1.1.jar:lib/log4j-1.2.15.jar:lib/commons-chain-1.1.jar
java -classpath .:$CP net.java.sampo.eijiro.EijiroFormatChecker $*
$ cat log4j.properties
log4j.rootCategory=WARN,A1
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
$ sh check.sh 108
2007-12-07 11:34:30,659 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｝" in "責任｛せきにん｝ある行動こうどう｝を取る"
2007-12-07 11:34:31,071 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "】" in "【分節】】a・pla・net・ic"
2007-12-07 11:34:31,565 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker ...
...
2007-12-07 11:34:38,792 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "(" in "超人的｛ちょうじん てき｝な(＝superhuman"
2007-12-07 11:34:39,139 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｛" in "無線｛むせん｝トランシーバ［送受信機｛そうじゅしんき］"
2007-12-07 11:34:39,272 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- 19 errors found, 1886646 lines read.

■act responsibly : 責任｛せきにん｝ある行動こうどう｝を取る、責任を持って行動｛する、自分の行動に責任を持つ
■aplanetic : 【発音】ae`plэne'tik、【分節】】a・pla・net・ic
■bow  {{3-自他動-1} : 〔物・体などを〕弓形｛ゆみがた｝に曲げる
■disagreeable : 【レベル】7、【＠】ディスアグリーアブル、デスアグリアブル、【変化】《複》disagreeables、\
【、【分節】dis・a・gree・a・ble
■Eisenhower  {人名} : アイゼンハワー◆ドワイト・アイゼンハワー(
■face-to-face talk : 直接｛ちょくせつ｝会談｛かいだん｝［交渉うこうしょう｝］、ひざ詰め談判｛づめ だんぱん｝、\
差し向かいの話し合い
■factor affecting : 〜に影響｛えいきょう｝する要因｛よういん［因子｛いんし｝］
■history of treatment : 治療（既往）歴（ちりょう（きおう）れき｝
■myelogenic : 【発音】】ma`iэlαdзe'nik、【＠】マイアラジェニク
■operating temperature : 作動｛さどう｝［運転｛うんてん｝・使用｛しよう・実用｛じつよう｝］温度｛おんど｝◆【略】OT
■pharmaceutical products : 医薬品｛いやくひん、｝、化学薬品｛かがく やくひん｝
■pulp fiction : 大衆文学｛たいしゅう、ぶんがく｝、大衆小説｛たいしゅう、しょうせつ｝、安っぽい小説｛しょうせつ｝、\
三文小説｛さんもん しょうせつ｝◆pulp（ざら紙）は安い。
■pulp novel : 大衆文学｛たいしゅう、ぶんがく｝、大衆小説｛たいしゅう、しょうせつ｝、安っぽい小説｛しょうせつ｝、\
三文小説｛さんもん しょうせつ｝◆pulp（ざら紙）は安い。
■release ~ into the bloodstream : 〜を血流｛けつりゅう｝［血液｛けつえき］中に放出｛ほうしゅつ｝する
■scrump  {自他動-2} : 米俗〉セックスする
■solution heat treatment : 固溶｛こよう［溶体｛ようたい｝］化熱処理｛か ねつしょり｝◆【略】SHT
■soman  {名} : 化》ソマン◆化学名は Pinacolyl methylphosphonofluoridate で神経ガスの一種◆【略】GD
■unhuman  {形-2} : 超人的｛ちょうじん てき｝な(＝superhuman
■wireless transceiver : 無線｛むせん｝トランシーバ［送受信機｛そうじゅしんき
$ 

$ sh check.sh 109
2007-12-07 12:25:59,984 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "】" in "【分節】】a・pla・net・ic"
2007-12-07 12:26:00,555 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "{" in "bow  {{3-自他動-1}"
2007-12-07 12:26:01,991 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "(" in "ドワイト・アイゼンハワー("
2007-12-07 12:26:02,002 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｝" in "電気｛でんき｝出力｛しゅつりょく｝［動力｛どうりょく｝｝"
2007-12-07 12:26:02,008 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｝" in "電気｛でんき｝出力｛しゅつりょく｝［動力｛どうりょく｝｝"
2007-12-07 12:26:04,711 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "】" in "【発音】】ma`iэlαdзe'nik"
2007-12-07 12:26:06,788 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "》" in "化》ソマン"
2007-12-07 12:26:07,717 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker 
- Unmatched "(" in "超人的｛ちょうじん てき｝な(＝superhuman"
2007-12-07 12:26:08,194 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- 8 errors found, 1899349 lines read.

■aplanetic : 【発音】ae`plэne'tik、【分節】】a・pla・net・ic
■bow  {{3-自他動-1} : 〔物・体などを〕弓形｛ゆみがた｝に曲げる
■Eisenhower  {人名} : アイゼンハワー◆ドワイト・アイゼンハワー(
■electric power : 電力｛でんりょく｝、電気｛でんき｝出力｛しゅつりょく｝［動力｛どうりょく｝｝◆【略】EP
■electrical power : 電力｛でんりょく｝、電気｛でんき｝出力｛しゅつりょく｝［動力｛どうりょく｝｝◆【略】EP
■myelogenic : 【発音】】ma`iэlαdзe'nik、【＠】マイアラジェニク
■soman  {名} : 化》ソマン◆化学名は Pinacolyl methylphosphonofluoridate で神経ガスの一種◆【略】GD
■unhuman  {形-2} : 超人的｛ちょうじん てき｝な(＝superhuman
$ 

import java.net.URL;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.fetcher.FeedFetcher;
import com.sun.syndication.fetcher.impl.HttpURLFeedFetcher;

public class KazMuzikBlogFeedFetcher {
  private static final String kazMuzikBlogFeedUrl = "http://kazuomik.livejournal.com/data/atom";

  public static void main(String[] args) throws Exception {
    FeedFetcher fetcher = new HttpURLFeedFetcher();
    SyndFeed feed = fetcher.retrieveFeed(new URL(kazMuzikBlogFeedUrl));
    System.out.println(feed.toString());
  }
} 

package net.java.sampo.eijiro;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.chain.Chain;
import org.apache.commons.chain.Command;
import org.apache.commons.chain.Context;
import org.apache.commons.chain.impl.ChainBase;
import org.apache.commons.chain.impl.ContextBase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class EijiroFormatChecker {

  private static final String eijiroFile = "EIJI-108.TXT";
  private static final String eijiroFileEncoding = "Shift-JIS";

  private static Log log = LogFactory.getLog(EijiroFormatChecker.class);

  private Context context;
  private Command checkLineCommand;

  public EijiroFormatChecker() throws IOException {
    context = new ContextBase();
    context.put("errors", new ArrayList<String>());
    checkLineCommand = new CheckLineCommand();
  }

  public void close() throws IOException {
    context = null;
  }

  public List<String> getErrorLines() {
    return (List<String>)context.get("errors");
  }

  public void checkFile() throws IOException {
    checkFile(eijiroFile, eijiroFileEncoding);
  }

  public void checkFile(String filename, String encoding) throws IOException {
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new FileInputStream(filename), encoding));
    int lineCount = 0;
    int errorCount = 0;
    if (log.isInfoEnabled()) {
      log.info("Started checking \"" + filename + "\" ...");
    }
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      context.put("line", line);
      try {
        checkLineCommand.execute(context);
      }
      catch (Exception e) {
        log.error(e.getMessage(), e);
      }

      lineCount ++;
      if (lineCount % 100000 == 0) {
        if (log.isInfoEnabled()) {
          log.info("" + lineCount + " lines processed.");
        }
      }
    }
    if (log.isInfoEnabled()) {
      log.info("" + lineCount + " lines processed, completed.");
      log.info("" + getErrorLines().size() + " errors found.");
    }
    in.close();
  }

  private static class CheckLineCommand extends ChainBase {
    public CheckLineCommand() {
      super();
      addCommand(new CheckSeparatorCommand(" : "));
      addCommand(new CheckParenthesisChain());
    }
  }

  private static class CheckSeparatorCommand implements Command {
    private String separator;

    public CheckSeparatorCommand(String separator) {
      this.separator = separator;
    }

    public boolean execute(Context context) {
      String line = (String)context.get("line");
      if (line == null) {
        log.error("line is null.");
        return true;
      }
      int n = line.indexOf(separator);
      if (n < 0) {
        ((List<String>)context.get("errors")).add(line);
        if (log.isWarnEnabled()) {
          log.warn("No separator \"" + separator+ "\" between English and Japanese - \""
                  + line + "\"");
        }
        return false;
      }
      return false;
    }
  }

  private static class CheckParenthesisChain extends ChainBase {
    public CheckParenthesisChain() {
      super();
      addCommand(new CheckParenthesisCommand('{', '}'));  // 品詞ラベル
      addCommand(new CheckParenthesisCommand('\u3010', '\u3011'));
                                     // 【】 特殊フィールド, スピーチラベル(?)
      addCommand(new CheckParenthesisCommand('\uff5b', '\uff5d'));  // ｛｝ 読み仮名
      addCommand(new CheckParenthesisCommand('\u300a', '\u300b'));  // 《》 分野ラベル
      addCommand(new CheckParenthesisCommand('\u3008', '\u3009'));  // 〈〉 分野ラベル
    }
  }

  private static class CheckParenthesisCommand implements Command {
    private char startChar;
    private char endChar;

    public  CheckParenthesisCommand(char startChar, char endChar) {
      this.startChar = startChar;
      this.endChar = endChar;
    }

    public boolean execute(Context context) {
      String line = (String)context.get("line");
      if (line == null) {
        log.error("line is null.");
        return true;
      }
      int n = 0;
      while (true) {
        n = line.indexOf(startChar, n);
        if (n < 0) {
          break;
        }
        int m = line.indexOf(endChar, n+1);
        if (m < 0) {
          ((List<String>)context.get("errors")).add(line);
          if (log.isWarnEnabled()) {
            log.warn("Unmatched \"" + startChar + "\" in \"" + line + "\"");
          }
          break;
        }
        if (m < 0 || m == line.length() - 1) {
          break;
        }
        n = m + 1;
      }
      return false;
    }
  }

  public static void main(String[] args) throws Exception {
    EijiroFormatChecker checker = new EijiroFormatChecker();
    checker.checkFile();
    System.out.println();
    for (String line : checker.getErrorLines()) {
      System.out.println(line);
    }
    checker.close();
  }
} 

$ cat check.sh
#!/bin/sh
CP=.:lib/eijiro-lucene.jar:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar\
:lib/commons-logging-1.1.1.jar:lib/log4j-1.2.15.jar:lib/commons-chain-1.1.jar
java -Dsen.home=/usr/java/sen -classpath $CP net.java.sampo.eijiro.EijiroFormatChecker
$ sh check.sh
2007-12-05 07:21:38,982 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - Started checking "EIJI-108.TXT" ...
2007-12-05 07:21:39,202 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｛" in "■act responsibly : 責任｛せきにん｝ある行動こうどう｝を取る、責任を持って行動｛する、自分の行動に責任を持つ"
2007-12-05 07:21:39,430 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 100000 lines processed.
2007-12-05 07:21:39,693 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 200000 lines processed.
2007-12-05 07:21:39,949 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 300000 lines processed.
2007-12-05 07:21:40,211 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 400000 lines processed.
2007-12-05 07:21:40,470 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 500000 lines processed.
2007-12-05 07:21:40,732 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 600000 lines processed.
2007-12-05 07:21:40,996 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 700000 lines processed.
2007-12-05 07:21:41,246 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 800000 lines processed.
2007-12-05 07:21:41,511 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 900000 lines processed.
2007-12-05 07:21:41,755 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1000000 lines processed.
2007-12-05 07:21:41,998 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1100000 lines processed.
2007-12-05 07:21:42,244 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1200000 lines processed.
2007-12-05 07:21:42,488 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1300000 lines processed.
2007-12-05 07:21:42,740 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1400000 lines processed.
2007-12-05 07:21:42,989 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1500000 lines processed.
2007-12-05 07:21:43,231 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1600000 lines processed.
2007-12-05 07:21:43,481 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1700000 lines processed.
2007-12-05 07:21:43,727 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1800000 lines processed.
2007-12-05 07:21:43,862 [main] WARN  net.java.sampo.eijiro.EijiroFormatChecker \
- Unmatched "｛" in "■wireless transceiver : 無線｛むせん｝トランシーバ［送受信機｛そうじゅしんき］"
2007-12-05 07:21:43,934 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 1886646 lines processed, completed.
2007-12-05 07:21:43,934 [main] INFO  net.java.sampo.eijiro.EijiroFormatChecker - 2 errors found.

■act responsibly : 責任｛せきにん｝ある行動こうどう｝を取る、責任を持って行動｛する、自分の行動に責任を持つ
■wireless transceiver : 無線｛むせん｝トランシーバ［送受信機｛そうじゅしんき］
$ 

$ cat index.sh
#!/bin/sh
CP=lib/eijiro-lucene.jar:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar
java -Dsen.home=/usr/java/sen -classpath $CP EijiroSimpleIndex $*
$ sh index.sh
> 天国への階段
Dec 4, 2007 4:41:48 PM net.java.sen.Dictionary 
INFO: token file = /usr/java/sen/dic/token.sen
Dec 4, 2007 4:41:48 PM net.java.sen.Dictionary 
INFO: time to load posInfo file = 8[ms]
Dec 4, 2007 4:41:48 PM net.java.sen.Dictionary 
INFO: double array trie dictionary = /usr/java/sen/dic/da.sen
Dec 4, 2007 4:41:48 PM net.java.sen.util.DoubleArrayTrie load
INFO: loading double array trie dict = /usr/java/sen/dic/da.sen
Dec 4, 2007 4:41:49 PM net.java.sen.util.DoubleArrayTrie load
INFO: loaded time = 0.141[ms]
Dec 4, 2007 4:41:49 PM net.java.sen.Dictionary 
INFO: pos info file = /usr/java/sen/dic/posInfo.sen
Dec 4, 2007 4:41:49 PM net.java.sen.Dictionary 
INFO: time to load pos info file = 0[ms]
Dec 4, 2007 4:41:49 PM net.java.sen.Tokenizer loadConnectCost
INFO: connection file = /usr/java/sen/dic/matrix.sen
Dec 4, 2007 4:41:49 PM net.java.sen.Tokenizer loadConnectCost
INFO: time to load connect cost file = 72[ms]
query=ja:"天国 へ の 階段"
1.000000[id=1858538] ■stairway to heaven : 〈楽曲〉天国への階段◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。...
> 
$ 

$ cat commons-logging.properties
org.apache.commons.logging.Log=org.apache.commons.logging.impl.Log4JLogger
$ cat log4j.properties
log4j.rootCategory=INFO,A1
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
$ cat index.sh
#!/bin/sh
CP=lib/eijiro-lucene.jar:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar
CP=.:$CP:lib/log4j-1.2.15.jar
java -Dsen.home=/usr/java/sen -classpath $CP EijiroSimpleIndex $*
$ sh index.sh
> 天国への階段
2007-12-04 17:23:31,136 [main] INFO  net.java.sen.Dictionary - token file = /usr/java/sen/dic/token.sen
2007-12-04 17:23:31,155 [main] INFO  net.java.sen.Dictionary - time to load posInfo file = 17[ms]
2007-12-04 17:23:31,155 [main] INFO  net.java.sen.Dictionary \
- double array trie dictionary = /usr/java/sen/dic/da.sen
2007-12-04 17:23:31,156 [main] INFO  net.java.sen.util.DoubleArrayTrie \
- loading double array trie dict = /usr/java/sen/dic/da.sen
2007-12-04 17:23:31,465 [main] INFO  net.java.sen.util.DoubleArrayTrie \
- loaded time = 0.309[ms]
2007-12-04 17:23:31,466 [main] INFO  net.java.sen.Dictionary - pos info file = /usr/java/sen/dic/posInfo.sen
2007-12-04 17:23:31,467 [main] INFO  net.java.sen.Dictionary - time to load pos info file = 1[ms]
2007-12-04 17:23:31,468 [main] INFO  net.java.sen.Tokenizer - connection file = /usr/java/sen/dic/matrix.sen
2007-12-04 17:23:31,617 [main] INFO  net.java.sen.Tokenizer - time to load connect cost file = 148[ms]
query=ja:"天国 へ の 階段"
1.000000[id=1858538] ■stairway to heaven : 〈楽曲〉天国への階段◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。...
> 
$ 

$ rm commons-logging.properties
$ cat log4j.properties
log4j.rootCategory=ERROR,A1
log4j.category.net.java.sen=INFO
log4j.category.net.java.sen.util=WARN
log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
$ sh index.sh
> 天国への階段
2007-12-04 17:36:57,312 [main] INFO  net.java.sen.Dictionary - token file = /usr/java/sen/dic/token.sen
2007-12-04 17:36:57,331 [main] INFO  net.java.sen.Dictionary - time to load posInfo file = 17[ms]
2007-12-04 17:36:57,332 [main] INFO  net.java.sen.Dictionary \
- double array trie dictionary = /usr/java/sen/dic/da.sen
2007-12-04 17:36:57,640 [main] INFO  net.java.sen.Dictionary - pos info file = /usr/java/sen/dic/posInfo.sen
2007-12-04 17:36:57,642 [main] INFO  net.java.sen.Dictionary - time to load pos info file = 2[ms]
2007-12-04 17:36:57,643 [main] INFO  net.java.sen.Tokenizer - connection file = /usr/java/sen/dic/matrix.sen
2007-12-04 17:36:57,791 [main] INFO  net.java.sen.Tokenizer - time to load connect cost file = 148[ms]
query=ja:"天国 へ の 階段"
1.000000[id=1858538] ■stairway to heaven : 〈楽曲〉天国への階段◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。...
> 
$ 

$ python
>>> from math import sqrt
>>> def d(w,h):
...  return sqrt(w*w+h*h)
... 
>>> def r(w,h,l):
...  return d(w,h) / l
... 
>>> d(1680,1050)
1981.1360377318867
>>> r(1680,1050,22)
90.051638078722121
>>> 
$ 

import net.java.sampo.lucene.eijiro.index.EijiroLuceneIndex;
import static net.java.sampo.lucene.eijiro.index.EijiroLuceneIndex.EijiroSearchResult;
import static net.java.sampo.lucene.eijiro.index.EijiroLuceneIndex.parse;
import org.apache.lucene.document.Document;

public class StairwayToHeaven {
  private static final String stairwayToHeaven
    = "天国への階段";
  private static final String lineStairwayToHeaven
    = "■stairway to heaven : "
    + "〈楽曲〉天国への階段"
    + "◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。"
    + "ジミー・ページとロバート・プラントによって作詞作曲された。"
    + "1971年に発表され、以後ロック史上最高の名曲の一つとして広く愛聴されている。"
    + "レコードでの演奏時間は約8分。";
  // http://ja.wikipedia.org/wiki/
  // %E5%A4%A9%E5%9B%BD%E3%81%B8%E3%81%AE%E9%9A%8E%E6%AE%B5_%28%E6%A5%BD%E6%9B%B2%29

  public static void main(String[] args) throws Exception {
    EijiroLuceneIndex index = new EijiroLuceneIndex("eijiro_simple_index3");
    index.openSearcher();
    EijiroSearchResult[] results = index.search("stairway AND heaven");
    boolean exists = false;
    for (EijiroSearchResult result : results) {
      String line = result.getLine();
      if (line.indexOf(stairwayToHeaven) >= 0) {
        exists = true;
      }
    }
    index.closeSearcher();
    if (exists) {
      System.err.println( "\"Stairway to Heaven\" already exists." );
    }
    else {
      System.err.println( "Adding \"Stairway to Heaven\"..." );
      index.openWriter();
      Document doc = parse(lineStairwayToHeaven);
      index.addDocument(doc);
      index.closeWriter(false);
    }
  }
}

$ java -Dsen.home=/usr/java/sen \
  -classpath .:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar \
  StairwayToHeaven
Adding "Stairway to Heaven" ...
$ sh index.sh 5
> stairway AND heaven
query=(+en:stairway +en:heaven)
1.000000[id=1576357] ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
1.000000[id=1858538] ■stairway to heaven : 〈楽曲〉天国への階段◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。\
ジミー・ページとロバート・プラントによって作詞作曲された。1971年に発表され、以後ロック史上最高の名曲の一つとして広く愛聴されている。\
レコードでの演奏時間は約8分。
> 天国への階段
query=ja:"天国 へ の 階段"
1.000000[id=1858538] ■stairway to heaven : 〈楽曲〉天国への階段◆イギリスのロックグループ、レッド・ツェッペリンの代表曲。\
ジミー・ページとロバート・プラントによって作詞作曲された。1971年に発表され、以後ロック史上最高の名曲の一つとして広く愛聴されている。\
レコードでの演奏時間は約8分。
> 
$ 

$ cd ~/kazmuzikblog
$ mkdir /usr/local/nutch-0.9/kazmuzik-url-dir
$ java -classpath classes LiveJournalMonthlyManager \
| cut -f 2 \
| sed -e 's/^/http\:\/\/kazuomik.livejournal.com\//' -e 's/$/.html/' \
> /usr/local/nutch-0.9/kazmuzik-url-dir/20071116.txt
$ cd /usr/local/nutch-0.9
$ bin/nutch inject kazmuzik-crawldb kazmuzik-url-dir
...
$ mkdir kazmuzik-segments
$ bin/nutch generate kazmuzik-crawldb kazmuzik-segments
...
Generator: segment: kazmuzik-segments/20071117161617
...
$ bin/nutch fetch kazmuzik-segments/20071117161617
...
fetching http://kazuomik.livejournal.com/141077.html
fetching http://kazuomik.livejournal.com/88261.html
fetching http://kazuomik.livejournal.com/151961.html
...
fetching http://kazuomik.livejournal.com/36988.html
fetching http://kazuomik.livejournal.com/79224.html
fetching http://kazuomik.livejournal.com/81466.html
Fetcher: done
$ bin/nutch readseg -list -dir kazmuzik-segments
NAME            GENERATED  FETCHER START        FETCHER END          FETCHED PARSED
20071117161617  609        2007-11-17T16:16:46  2007-11-17T17:11:29  609     594
$ touch kazmuzik-segments/20071117161617/fetcher.done
$ bin/nutch updatedb kazmuzik-crawldb -dir kazmuzik-segments -noAdditions
...
$ bin/nutch readdb kazmuzik-crawldb -stats
CrawlDb statistics start: kazmuzik-crawldb
Statistics for CrawlDb: kazmuzik-crawldb
TOTAL urls:     609
retry 0:        609
min score:      1.001
avg score:      1.015
max score:      1.114
status 1 (db_unfetched):        15
status 2 (db_fetched):  594
CrawlDb statistics: done
$ bin/nutch generate kazmuzik-crawldb kazmuzik-segments
...
Generator: segment: kazmuzik-segments/20071117184651
...
$ bin/nutch fetch kazmuzik-segments/20071117184651
...
fetching http://kazuomik.livejournal.com/152532.html
fetching http://kazuomik.livejournal.com/148363.html
fetching http://kazuomik.livejournal.com/34221.html
Fetcher: done
$ bin/nutch readseg -list -dir kazmuzik-segments
NAME            GENERATED  FETCHER START        FETCHER END          FETCHED PARSED
20071117161617  609        2007-11-17T16:16:46  2007-11-17T17:11:29  609     594
20071117184651  15         2007-11-17T18:47:24  2007-11-17T18:48:40  15      15
$ bin/nutch updatedb kazmuzik-crawldb -dir kazmuzik-segments -noAdditions
...
$ bin/nutch readdb kazmuzik-crawldb -stats
CrawlDb statistics start: kazmuzik-crawldb
Statistics for CrawlDb: kazmuzik-crawldb
TOTAL urls:     609
retry 0:        609
min score:      1.002
avg score:      1.031
max score:      1.228
status 2 (db_fetched):  609
CrawlDb statistics: done
$ bin/nutch mergesegs kazmuzik-segments2 -dir kazmuzik-segments
...
$ bin/nutch readseg -list -dir kazmuzik-segments2
NAME            GENERATED  FETCHER START        FETCHER END          FETCHED PARSED
20071117185404  609        2007-11-17T16:16:46  2007-11-17T18:48:40  609     609
$ cd ~/kazmuzikblog
$ java -classpath classes:/usr/java/jdk/db/lib/derby.jar:/usr/local/nutch-0.9/nutch-0.9.jar:\
/usr/local/nutch-0.9/lib/hadoop-0.12.2-core.jar:/usr/local/nutch-0.9/lib/commons-logging-1.0.4.jar:\
/usr/local/nutch-0.9/lib/log4j-1.2.13.jar \
LiveJournalEntryDatabaseInitializer /usr/local/nutch-0.9/kazmuzik-segments2/20071117185404
...
$ 

$ cd /usr/local/nutch-0.9
$ mkdir kazmuzik-url-dir2
$ cat > kazmuzik-url-dir2/20071116b.txt
http://kazuomik.livejournal.com/128843.html
http://kazuomik.livejournal.com/129431.html
http://kazuomik.livejournal.com/152816.html
^D
$ bin/nutch freegen kazmuzik-url-dir2 kazmuzik-segments2
...
$ bin/nutch fetch kazmuzik-segments2/20071117221341
...
$ bin/nutch mergesegs kazmuzik-segments3 -dir kazmuzik-segments2
...
$ bin/nutch readseg -list -dir kazmuzik-segments3
NAME            GENERATED  FETCHER START        FETCHER END          FETCHED PARSED
20071117221515  609        2007-11-17T16:16:46  2007-11-17T22:14:35  609     609
$ java -classpath classes:/usr/java/jdk/db/lib/derby.jar:/usr/local/nutch-0.9/nutch-0.9.jar:\
/usr/local/nutch-0.9/lib/hadoop-0.12.2-core.jar:/usr/local/nutch-0.9/lib/commons-logging-1.0.4.jar:\
/usr/local/nutch-0.9/lib/log4j-1.2.13.jar \
LiveJournalEntryDatabaseInitializer /usr/local/nutch-0.9/kazmuzik-segments3/20071117221515
$ java -classpath classes:/usr/java/derby/lib/derby.jar LiveJournalHtmlCreator
$ 

$ cat index.sh
#!/bin/sh
CP=lib/eijiro-lucene.jar:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar
java -Dsen.home=/usr/java/sen -classpath $CP EijiroSimpleIndex $*
$ sh index.sh 3
> stairway to heaven
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
0.464318 ■stairway  {名} : 階段｛かいだん｝、階段室｛かいだんしつ｝\
          ■・Tom runs up and down the stairway to get exercise.\
            トムは運動のために階段を走って上り下りする。
0.464318 ■stairway : 【レベル】9、【＠】ステアウェー、ステアウェイ、【変化】《複》stairways、【分節】stair・way
> 天国への階段
(no result)
> 天国 階段
1.000000 ■blessed land : 天国｛てんごく｝
1.000000 ■divine Kingdom : 天国｛てんごく｝
1.000000 ■happy land : 天国｛てんごく｝
> 
$

$ sh index.sh 1
> stairway to heaven  
query=(en:stairway en:heaven)
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
> stairway OR heaven
query=(en:stairway en:heaven)
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
> stairway AND heaven
query=(+en:stairway +en:heaven)
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
> "stairway to heaven"
query=en:"stairway heaven"
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
> 天国への階段
query=ja:"天国 へ の 階段"
(no result)
> 天国 階段
query=(ja:天国 ja:階段)
1.000000 ■blessed land : 天国｛てんごく｝
> 天国 OR 階段
query=(ja:天国 ja:階段)
1.000000 ■blessed land : 天国｛てんごく｝
> 天国 AND 階段
query=(+ja:天国 +ja:階段)
(no result)
> 天国への
query=ja:"天国 へ の"
1.000000 ■Al Sirat : 宗教上の正道、天国への橋
> パンストの伝線
query=ja:"パン スト の 伝 線"
1.000000 ■ladder  {名-3} : 〈英〉〔パンストの〕伝線｛でんせん｝◆【同】〈米〉run
> パンスト伝線
query=ja:"パン スト 伝 線"
(no result)
> パンスト 伝線
query=(ja:"パン スト" ja:"伝 線")
1.000000 ■ladder  {名-3} : 〈英〉〔パンストの〕伝線｛でんせん｝◆【同】〈米〉run
> パンスト AND 伝線
query=(+ja:"パン スト" +ja:"伝 線")
1.000000 ■ladder  {名-3} : 〈英〉〔パンストの〕伝線｛でんせん｝◆【同】〈米〉run
>
$

$ ant
$ cat monthly.sh
#!/bin/sh
java -classpath classes LiveJournalMonthlyManager \
| cut -f 2 \
> tmp1.txt
paste -d '#' tmp1.txt tmp1.txt \
| sed -e 's/^/\<a href\=\"\//' -e 's/\#/\.html\"\>/' -e 's/$/\<\/a\>/' \
> tmp2.txt
$ sh monthyly.sh
$ cat tmp2.txt
<a href="/2603.html">2603</a>
<a href="/2080.html">2080</a>
<a href="/2409.html">2409</a>
...
<a href="/151961.html">151961</a>
<a href="/145303.html">145303</a>
<a href="/152249.html">152249</a>
$ wc -l tmp2.txt
594 tmp2.txt
$

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Calendar;
import java.util.Date;

public class LiveJournalMonthlyParser extends BufferedReader {

  private int yyyy;
  private int mm;
  private int dd;

  public LiveJournalMonthlyParser(Reader in) throws IOException {
    super(in);
  }

  public void close() throws IOException {
    super.close();
  }

  public LiveJournalEntry parse() throws IOException {
    int mode = 0;
    while (true) {
      String line = readLine();
      if (line == null) {
        break;
      }
      if (mode == 1) {
        int n = line.indexOf(".livejournal.com/");
        yyyy = Integer.parseInt( line.substring(n+17, n+21) );
        mm   = Integer.parseInt( line.substring(n+22, n+24) );
        dd   = Integer.parseInt( line.substring(n+25, n+27) );
        mode = 0;
        continue;
      }
      if (mode == 2) {
        int n = line.indexOf(':');
        int _hh = Integer.parseInt( line.substring(n-2,n) );
        int _mm = Integer.parseInt( line.substring(n+1,n+3) );
        if (line.charAt(n+4) == 'p') {
          _hh += 12;
        }
        Calendar cal = Calendar.getInstance();
        cal.set(yyyy, mm-1, dd, _hh, _mm);
        Date date = cal.getTime();
        n = line.indexOf(".livejournal.com/");
        int m = line.indexOf('.', n+17);
        int id = Integer.parseInt( line.substring(n+17,m) );
        n = line.indexOf("</a>", m+7);
        String title = line.substring(m+7,n);
        LiveJournalEntry entry = new LiveJournalEntry(id);
        entry.setTitle(title);
        entry.setDate(date);
        return entry;
      }
      if (line.indexOf("class=\"entryDate\"") >= 0) {
        mode = 1;
        continue;
      }
      if (line.indexOf("class=\"entry\"") >= 0) {
        mode = 2;
        continue;
      }
    }
    return null;
  }
}

import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Formatter;
import java.util.List;

public class LiveJournalMonthlyManager {
  public LiveJournalMonthlyManager() {
  }

  public List getEntries(int yyyy, int mm)
      throws IOException {
    String urlString = getURLString(yyyy, mm);
    URL url = new URL( getURLString(yyyy, mm) );
    InputStream is = null;
    try {
      is = url.openStream();
    }
    catch (Exception e) {
      System.err.printf("! %s : %s%n", urlString, e.getMessage());
      return null;
    }
    if (is == null) {
      return null;
    }
    LiveJournalMonthlyParser parser
      = new LiveJournalMonthlyParser(new InputStreamReader(is, "UTF-8"));
    List list = new ArrayList();
    while (true) {
      LiveJournalEntry entry = parser.parse();
      if (entry == null) {
        break;
      }
      list.add(entry);
    }
    parser.close();
    return list;
  }

  private static String getURLString(int yyyy, int mm) {
    Formatter f = new Formatter();
    f.format("http://kazuomik.livejournal.com/%04d/%02d/", yyyy, mm);
    return f.toString();
  }
  
  public static void main(String[] args) throws Exception {
    LiveJournalMonthlyManager manager = new LiveJournalMonthlyManager();
    int yyyy = 2006;
    int mm = 9;
    while (true) {
      List list = manager.getEntries(yyyy, mm);
      if (list == null || list.size() < 1) {
        break;
      }
      for (LiveJournalEntry entry : list) {
        Date time = entry.getDate();
        System.out.printf("%tF %tR\t%d\t%s%n",
          time, time, entry.getId(), entry.getTitle());
      }
      if (++mm > 12) {
        yyyy++;
        mm -= 12;
      }
    }
  }
}

public class LiveJournalEntryParser extends BufferedReader {
  ...
  public LiveJournalEntry parse() throws IOException {
    ...
    String location = null;
    m = line.indexOf("<strong>Current Mood:</strong> ");
    if (m >= 0) {
      mood = removeTags( line.substring(m+31) );
      line = line.substring(0,m);
    }
    m = line.indexOf("<strong>Current Location:</strong> ");
    if (m >= 0) {
      location = removeTags( line.substring(m+35) );
      line = line.substring(0,m);
    }
    m = line.indexOf("<strong>Tags:</strong> ");
    ...
  }
  ...
}

$ ant
$ java -classpath classes:/usr/java/derby/lib/derby.jar　LiveJournalFetchPreparator -init
http://kazuomik.livejournal.com/
152249*I 145303*I 151961*I 151707*I 151514*I 135539*I 151218*I 150763*I 150220*I 149897*I
149594*I 151028*I 150495*I 149305*I 149228*I 148921*I 148134*I 148649*I 148363*I 147935*I
http://kazuomik.livejournal.com/?skip=20
...
http://kazuomik.livejournal.com/?skip=380
54835*I 54734*I 54482*I 54085*I 53978*I 53663*I 53377*I 53139*I 52777*I 52543*I
52328*I 52168*I 51855*I 51556*I 51418*I 51044*I 50862*I 50481*I 50411*I 50032*I
http://kazuomik.livejournal.com/?skip=400
54835 54734 54482 54085 53978 53663 53377 53139 52777 52543
52328 52168 51855 51556 51418 51044 50862 50481 50411 50032
$ 

import java.awt.BorderLayout;
import static java.awt.BorderLayout.CENTER;
import static java.awt.BorderLayout.EAST;
import static java.awt.BorderLayout.NORTH;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.io.IOException;
import java.util.Formatter;
import javax.swing.JButton;
import javax.swing.JFrame;
import static javax.swing.JFrame.EXIT_ON_CLOSE;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.swing.JTextField;
import org.apache.lucene.queryParser.ParseException;

public class EijiroLucenePanel extends JPanel implements ActionListener {
  private EijiroSimpleIndex index;

  private JTextField queryField;
  private JButton searchButton;
  private JTextArea resultArea;

  public EijiroLucenePanel(EijiroSimpleIndex index) {
    super();
    setLayout(new BorderLayout());

    this.index = index;

    JPanel searchPanel = new JPanel();
    add(searchPanel, NORTH);
    searchPanel.setLayout(new BorderLayout());

    queryField = new JTextField(32);
    searchPanel.add(queryField, CENTER);
    queryField.addActionListener(this);

    searchButton = new JButton("Search");
    searchPanel.add(searchButton, EAST);
    searchButton.addActionListener(this);

    resultArea = new JTextArea(24, 80);
    add(new JScrollPane(resultArea), CENTER);
  }

  public void actionPerformed(ActionEvent ae) {
    String query = queryField.getText();
    EijiroSimpleIndex.EijiroSearchResult[] results = null;
    try {
      results = index.search(query);
    }
    catch (IOException ex) {
      System.err.println(ex.getMessage());
      return;
    }
    catch (ParseException ex) {
      System.err.println(ex.getMessage());
      return;
    }
    resultArea.setText("");
    for (EijiroSimpleIndex.EijiroSearchResult result : results) {
      Formatter f = new Formatter();
      f.format("%f %s%n", result.getScore(), result.getLine());
      resultArea.append(f.toString());
    }
  }

  public static void main(String[] args) throws Exception {
    JFrame frame = new JFrame("Eijiro Lucene");
    frame.setLayout(new BorderLayout());
    frame.add(new EijiroLucenePanel(new EijiroSimpleIndex()), CENTER);
    frame.setDefaultCloseOperation(EXIT_ON_CLOSE);
    frame.pack();
    frame.setVisible(true);
  }
}

$ java -Dsen.home=/usr/java/sen \
  -classpath .:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar \
  EijiroSimpleIndex 10
> heaven
1.000000 ■heaven  {間投} : 《heavens》これは大変！、まあ！、何てことだ！、とんでもない！\
          ■・Heavens! One of the biggest typhoons is coming. これは大変だ。最大級の台風が来てるって。
1.000000 ■heaven  {名-1} : 天国｛てんごく｝、神、天■・When I die I want to go to heaven. 死んだら天国へ行きたい。
1.000000 ■heaven  {名-2} : 至上｛しじょう｝の幸福｛こうふく｝、素晴｛すば｝らしいもの
1.000000 ■Heaven's will : 天命｛てんめい｝、天寿｛てんじゅ｝、神のご意志｛いし｝
1.000000 ■in heaven  {1} : 《be ～》天国にいる、死んでいる
1.000000 ■in heaven  {2} : 《be ～》天にも昇る気分｛きぶん｝［ような気持ち］である
1.000000 ■in heaven : 天に、天国に、死んで
1.000000 ■to heaven : ぜひとも〔～できればと願う〕\
          ■・I wish to (high) heaven I were with you right now. 今あなたのそばにいられたら、どんなにかよかったのに。
1.000000 ■will of Heaven : 《the ～》神のご意志｛いし｝、天命｛てんめい｝
0.625000 ■ascend to heaven : 天に昇る、天国｛てんごく｝に行く
> 天国
1.000000 ■blessed land : 天国｛てんごく｝
1.000000 ■divine Kingdom : 天国｛てんごく｝
1.000000 ■happy land : 天国｛てんごく｝
1.000000 ■kingdom of Heaven : 天国｛てんごく｝
1.000000 ■Land of the Leal : 天国｛てんごく｝
1.000000 ■sky  {名-2} : 天国｛てんごく｝
0.625000 ■on the other side : 天国｛てんごく｝で\
          ■・I will talk to my grandfather again when we meet on the other side. もし天国で祖父に会ったら、また話をしたい。
0.625000 ■paradise for criminals : 犯罪天国｛はんざい てんごく｝
0.625000 ■paradise of fashion : ファッション天国｛てんごく｝
0.625000 ■paradisiacal  {形} : 天国｛てんごく｝の
> 天国 AND 神 AND 天
1.000000 ■heaven  {名-1} : 天国｛てんごく｝、神、天■・When I die I want to go to heaven. 死んだら天国へ行きたい。
> 
$ 

$ java -Dsen.home=/usr/java/sen \
  -classpath .:lib/lucene-core-2.2.0.jar:lib/lucene-ja.jar:lib/sen.jar:lib/commons-logging.jar \
  EijiroSimpleIndex 0
> 天国
1.000000 ■blessed land : 天国｛てんごく｝
1.000000 ■divine Kingdom : 天国｛てんごく｝
1.000000 ■happy land : 天国｛てんごく｝
1.000000 ■kingdom of Heaven : 天国｛てんごく｝
1.000000 ■Land of the Leal : 天国｛てんごく｝
1.000000 ■sky  {名-2} : 天国｛てんごく｝
0.625000 ■on the other side : 天国｛てんごく｝で\
          ■・I will talk to my grandfather again when we meet on the other side. もし天国で祖父に会ったら、また話をしたい。
...
0.500000 ■go to the better place : 死ぬ、天国｛てんごく｝に行く
0.500000 ■head off to heaven : 天国｛てんごく｝に向かう、死ぬ
0.500000 ■heaven  {名-1} : 天国｛てんごく｝、神、天\
          ■・When I die I want to go to heaven. 死んだら天国へ行きたい。
0.500000 ■heaven and hell : 天国｛てんごく｝と地獄｛じごく｝
0.500000 ■heaven on earth : 楽園｛らくえん｝、地上｛ちじょう｝の天国｛てんごく｝
...
0.250000 ■with God : 死んで神のみもとに、神とともにある、死んで天国にいる
> ツェッペリン
1.000000 ■Count Ferdinand von Zeppelin  {人名} : ツェッペリン◆1838-1917。ドイツの軍人。第二次世界大戦前からの飛行船設計、製造で有名。
0.618718 ■Graf Zeppelin : グラーフ・ツェッペリン、ツェッペリン伯号
0.500000 ■Ferdinand Graf von Zeppelin  {人名} : フェルディナンド・グラフ・ツェッペリン\
          ◆1838-1917。ドイツノ軍人であり発明家。飛行船を発明したことで知られる。
0.500000 ■Zeppelin  {名} : 《航空》ツェッペリン型飛行船\
          ◆ドイツの大型飛行船。第一次大戦でロンドンを爆撃し、空の要塞｛ようさい｝と呼ばれた。\
            第二次世界大戦前は米の戦略的ヘリウム禁輸出で、水素ガスを使用。その為墜落事故を起した。
0.375000 ■Song Remains the Same  {映画} : 《The ～》レッド・ツェッペリン／狂熱のライブ◆米1976
> レッドツェッペリン
1.000000 ■Led Zeppelin : レッドツェッペリン◆英国のロックグループ
> 
$ 

...

public class EijiroSimpleIndex {
  private static final String defaultIndexDirName
    = "eijiro_simple_index3";

  private static final String eijiroFile = "eiji-108a.txt";
  private static final String encoding = "UTF-8";

  ...

  private static Document parse(String line) {
    Document doc = new Document();
    doc.add(new Field("line", line, Field.Store.COMPRESS, Field.Index.NO));
    int n = line.indexOf(" : ");
    if (n < 0) {
      return null;
    }
    String en = line.substring(1, n);
    String ja = line.substring(n+3);
    en = removeParenthesis(en, '{', '}').trim(); // 品詞ラベル
    doc.add(new Field("en", en, Field.Store.NO, Field.Index.TOKENIZED));

    n = ja.indexOf("\u25c6"); // ◆
    if (n >= 0) {
      ja = ja.substring(0,n); // 補足説明
    }
    else {
      n = ja.indexOf("\u25a0\u30fb"); // ■・
      if (n >= 0) {
        ja = ja.substring(0,n); // 文例
      }
    }
    ja = ja.trim();
    n = 0;
    while (true) {
      int m = ja.indexOf('、', n);
      if (m < 0) {
        addJapaneseField(doc, ja.substring(n));
        break;
      }
      addJapaneseField(doc, ja.substring(n, m));
      n = m+1;
    }
    Field[] fields = doc.getFields("ja");
    if (fields == null || fields.length < 1) {
      return null;
    }
    return doc;
  }

  private static void addJapaneseField(Document doc, String s) {
    s = normalizeJapanese(s);
    if (s == null) {
      return;
    }
    doc.add(new Field("ja", s, Field.Store.NO, Field.Index.TOKENIZED));
  }

  private static String normalizeJapanese(String s) {
    String s1 = removeParenthesis(s, '【', '】'); // 特殊フィールド
                                                  // スピーチラベル?
    if (s1.length() != s.length()) {
      return null;
    }
    s = removeParenthesis(s, '｛', '｝'); // 読み仮名
    s = removeParenthesis(s, '《', '》'); // 分野ラベル
    s = removeParenthesis(s, '〈', '〉'); // 分野ラベル
    return s;
  }

  private static String removeParenthesis(String s,
                          char startChar, char endChar) {
    String s0 = s;
    int n = 0;
    while (true) {
      n = s.indexOf(startChar, n);
      if (n < 0) {
        break;
      }
      int m = s.indexOf(endChar, n+1);
      if (m < 0) {
        System.err.printf("ERROR: %s%n", s0);
      }
      if (m < 0 || m == s.length() - 1) {
        s = s.substring(0,n);
        break;
      }
      s = s.substring(0,n) + s.substring(m+1);
    }
    return s;
  }

  public static class EijiroSearchResult {
    private float score;
    private String line;

    public EijiroSearchResult(float score, String line) {
      this.score = score;
      this.line = line;
    }

    public float getScore() {
      return score;
    }

    public String getLine() {
      return line;
    }
  }

  public EijiroSearchResult[] search(String q)
      throws IOException, ParseException {
    return search(q, 0);
  }

  public EijiroSearchResult[] search(String q, int n)
      throws IOException, ParseException {
    if (containsNonAsciiChar(q)) {
      return search(q, n, false, true);
    }
    else {
      return search(q, n, true, false);
    }
  }

  private static boolean containsNonAsciiChar(String s) {
    int n = s.length();
    for (int i = 0; i < n; i++) {
      if (s.charAt(i) >= '\u0080') {
        return true;
      }
    }
    return false;
  }

  public EijiroSearchResult[] search(String q, int n,
                                     boolean en, boolean ja)
      throws IOException, ParseException {
    BooleanQuery query = new BooleanQuery();
    if (en) {
      query.add(enParser.parse(q), BooleanClause.Occur.SHOULD);
    }
    if (ja) {
      query.add(jaParser.parse(q), BooleanClause.Occur.SHOULD);
    }

    Hits hits = searcher.search(query);
    int len = hits.length();
    if (n > 0 && len > n) {
      len = n;
    }
    EijiroSearchResult[] results = new EijiroSearchResult[len];
    for (int i = 0; i < len; i++) {
      Document doc = hits.doc(i);
      results[i] = new EijiroSearchResult(hits.score(i), doc.get("line"));
    }
    return results;
  }

  public static void main(String[] args) throws Exception {
    int maxCount = 10;
    if (args.length > 0) {
      try {
        maxCount = Integer.parseInt(args[0]);
      }
      catch (NumberFormatException e) {
      }
    }
    EijiroSimpleIndex index = new EijiroSimpleIndex();
    BufferedReader in
      = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
    while (true) {
      System.out.print("> ");
      System.out.flush();
      String q = in.readLine();
      if (q == null) {
        break;
      }
      q = q.trim();
      if (q.length() < 1) {
        continue;
      }
      EijiroSearchResult[] results = index.search(q, maxCount);
      if (results == null || results.length < 1) {
        System.out.println("(no result)");
        continue;
      }
      for (EijiroSearchResult result : results) {
        System.out.printf("%f %s%n", result.getScore(), result.getLine());
      }
    }
    System.out.println();
    in.close();
    index.close();
  }
}

import org.apache.lucene.analysis.ja.JapaneseAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Searcher;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;

public class EijiroSimpleIndex {
  private static final String defaultIndexDirName
    = "eijiro_simple_index2";

  private static final String eijiroFile = "EIJI-108.TXT";
  private static final String encoding = "Shift-JIS";

  private static final int MAX_RESULTS = 10;

  private File indexDirName;
  private IndexReader reader;
  private Searcher searcher;
  private QueryParser enParser;
  private QueryParser jaParser;

  public EijiroSimpleIndex() throws IOException {
    this(defaultIndexDirName);
  }

  public EijiroSimpleIndex(String indexDirName) throws IOException {
    this.indexDirName = new File(indexDirName);
    if (! (new File(indexDirName).exists())) {
      createIndex();
    }

    reader = IndexReader.open(indexDirName);
    searcher = new IndexSearcher(reader);
    enParser = new QueryParser("en", new StandardAnalyzer());
    jaParser = new QueryParser("ja", new JapaneseAnalyzer());
  }

  public void close() throws IOException {
    reader.close();
  }

  public void createIndex() throws IOException {
    IndexWriter writer
      = new IndexWriter(indexDirName, new JapaneseAnalyzer());
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new FileInputStream(eijiroFile), encoding));
    int count = 0;
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      Document document = parse(line);
      if (document == null) {
        continue;
      }
      writer.addDocument(document);

      count ++;
      if (count >= 1000) {
        System.err.print(".");
        count = 0;
      }
    }
    in.close();
    System.err.println();
    writer.optimize();
    writer.close();
  }

  private static Document parse(String line) {
    Document doc = new Document();
    doc.add(new Field("line", line, Field.Store.COMPRESS, Field.Index.NO));
    int n = line.indexOf(" : ");
    if (n < 0) {
      return null;
    }
    String en = line.substring(1, n);
    String ja = line.substring(n+3);
    n = en.lastIndexOf("}");
    if (n >= 0) {
      int m = en.lastIndexOf(" {", n-1);
      en = en.substring(0, m); // 品詞 removed
    }
    en = en.trim();
    doc.add(new Field("en", en, Field.Store.NO, Field.Index.TOKENIZED));
    n = ja.indexOf("\u25c6"); // ◆
    if (n >= 0) {
      ja = ja.substring(0,n); // 補足説明 removed
    }
    else {
      n = ja.indexOf("\u25a0\u30fb"); // ■・
      if (n >= 0) {
        ja = ja.substring(0,n); // 文例 removed
      }
    }
    ja = ja.trim();
    doc.add(new Field("ja", ja, Field.Store.NO, Field.Index.TOKENIZED));
    return doc;
  }

  public static class EijiroSearchResult {
    private float score;
    private String line;

    public EijiroSearchResult(float score, String line) {
      this.score = score;
      this.line = line;
    }

    public float getScore() {
      return score;
    }

    public String getLine() {
      return line;
    }
  }

  public EijiroSearchResult[] search(String q)
      throws IOException, ParseException {
    if (containsNonAsciiChar(q)) {
      return search(q, false, true);
    }
    else {
      return search(q, true, false);
    }
  }

  private static boolean containsNonAsciiChar(String s) {
    int n = s.length();
    for (int i = 0; i < n; i++) {
      if (s.charAt(i) >= '\u0080') {
        return true;
      }
    }
    return false;
  }

  public EijiroSearchResult[] search(String q, boolean en, boolean ja)
      throws IOException, ParseException {
    BooleanQuery query = new BooleanQuery();
    if (en) {
      query.add(enParser.parse(q), BooleanClause.Occur.SHOULD);
    }
    if (ja) {
      query.add(jaParser.parse(q), BooleanClause.Occur.SHOULD);
    }

    Hits hits = searcher.search(query);
    int n = hits.length();
    if (n > MAX_RESULTS) {
      n = MAX_RESULTS;
    }
    EijiroSearchResult[] results = new EijiroSearchResult[n];
    for (int i = 0; i < n; i++) {
      Document doc = hits.doc(i);
      results[i] = new EijiroSearchResult(hits.score(i), doc.get("line"));
    }
    return results;
  }

  public static void main(String[] args) throws Exception {
    EijiroSimpleIndex index = new EijiroSimpleIndex();
    BufferedReader in
      = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
    while (true) {
      System.out.print("> ");
      System.out.flush();
      String q = in.readLine();
      if (q == null) {
        break;
      }
      q = q.trim();
      if (q.length() < 1) {
        continue;
      }
      EijiroSearchResult[] results = index.search(q);
      if (results == null || results.length < 1) {
        System.out.println("(no result)");
        continue;
      }
      int count = 0;
      for (EijiroSearchResult result : results) {
        System.out.printf("%f %s%n", result.getScore(), result.getLine());
        count ++;
        if (count >= 5) {
          break;
        }
      }
    }
    System.out.println();
    in.close();
    index.close();
  }
}

$ javac -classpath /usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-ja/lib/lucene-ja.jar \
  EijiroSimpleIndex.java
$ java -Dsen.home=/usr/java/sen \
-classpath .:/usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-ja/lib/lucene-ja.jar:\
/usr/java/sen/lib/sen.jar:/usr/java/sen/lib/commons-logging.jar EijiroSimpleIndex
...
> heaven
2007/10/30 01:23:45 net.java.sen.Dictionary 
...
情報: time to load connect cost file = 128[ms]
1.000000 ■heaven born  {形} : <→heaven-born>
1.000000 ■heaven sent  {形} : <→heaven-sent>
1.000000 ■tea of heaven  {名} : <→tea-of-heaven>
0.857143 ■heaven appointed task : <→heaven-appointed task>
0.857143 ■heaven kissing mountains : <→heaven-kissing mountains>
> stairway
1.000000 ■ladder like stairway : <→ladder-like stairway>
0.824958 ■ascend the stairway : 階段を上る
0.824958 ■climb a stairway : 階段を上る
0.824958 ■grimy stairway : 汚れた階段
0.824958 ■mount a stairway : 階段を上る
> stairway to heaven
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
0.621393 ■ladder like stairway : <→ladder-like stairway>
0.512623 ■ascend the stairway : 階段を上る
0.512623 ■climb a stairway : 階段を上る
0.512623 ■grimy stairway : 汚れた階段
> stairway AND heaven
1.000000 ■stairway to heaven : 〈豪俗〉（パンストの）伝線◆【同】run ; ladder
>
$ du -k -s eijiro_simple_index
329136  eijiro_simple_index
$

import org.apache.lucene.analysis.ja.JapaneseAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Searcher;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;

public class EijiroSimpleIndex {
  private static final String defaultIndexDirName
    = "eijiro_simple_index";

  private static final String eijiroFile = "EIJI-108.TXT";
  private static final String encoding = "Shift-JIS";

  private static final int MAX_RESULTS = 5;

  private File indexDirName;
  private IndexReader reader;
  private Searcher searcher;
  private QueryParser parser;

  public EijiroSimpleIndex() throws IOException {
    this(defaultIndexDirName);
  }

  public EijiroSimpleIndex(String indexDirName) throws IOException {
    this.indexDirName = new File(indexDirName);
    if (! (new File(indexDirName).exists())) {
      createIndex();
    }

    reader = IndexReader.open(indexDirName);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser("line", new JapaneseAnalyzer());
  }

  public void close() throws IOException {
    reader.close();
  }

  public void createIndex() throws IOException {
    IndexWriter writer
      = new IndexWriter(indexDirName, new JapaneseAnalyzer());
    BufferedReader in
      = new BufferedReader(new InputStreamReader(
          new FileInputStream(eijiroFile), encoding));
    int count = 0;
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      Document document = new Document();
      Field field = new Field("line", line, Field.Store.COMPRESS,
                              Field.Index.TOKENIZED);
      document.add(field);
      writer.addDocument(document);

      count ++;
      if (count >= 1000) {
        System.err.print(".");
        count = 0;
      }
    }
    in.close();
    System.err.println();
    writer.optimize();
    writer.close();
  }

  public static class EijiroSearchResult {
    private float score;
    private String line;

    public EijiroSearchResult(float score, String line) {
      this.score = score;
      this.line = line;
    }

    public float getScore() {
      return score;
    }

    public String getLine() {
      return line;
    }
  }

  public EijiroSearchResult[] search(String q)
      throws IOException, ParseException {
    Query query = parser.parse(q);
    Hits hits = searcher.search(query);
    int n = hits.length();
    if (n > MAX_RESULTS) {
      n = MAX_RESULTS;
    }
    EijiroSearchResult[] results = new EijiroSearchResult[n];
    for (int i = 0; i < n; i++) {
      Document doc = hits.doc(i);
      results[i] = new EijiroSearchResult(hits.score(i), doc.get("line"));
    }
    return results;
  }

  public static void main(String[] args) throws Exception {
    EijiroSimpleIndex index = new EijiroSimpleIndex();
    BufferedReader in
      = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
    while (true) {
      System.out.print("> ");
      System.out.flush();
      String q = in.readLine();
      if (q == null) {
        break;
      }
      q = q.trim();
      if (q.length() < 1) {
        continue;
      }
      EijiroSearchResult[] results = index.search(q);
      if (results == null || results.length < 1) {
        System.out.println("(no result)");
        continue;
      }
      for (EijiroSearchResult result : results) {
        System.out.printf("%f %s%n", result.getScore(), result.getLine());
      }
    }
    System.out.println();
    in.close();
    index.close();
  }
}

$ iconv -f SHIFT-JIS -t UTF-8 EIJI-108.TXT > eiji-108.utf8.txt
iconv: illegal input sequence at position 185586
$ iconv -f SJIS-WIN -t UTF-8 EIJI-108.TXT > eiji-108.utf8.txt
$ wc -l EIJI-108.TXT eiji-108.utf8.txt
  1886646 EIJI-108.TXT
  1886646 eiji-108.utf8.txt
  3773292 total
$

import org.apache.lucene.index.memory.SynonymMap;
import java.io.FileInputStream;

public class SynonymMapTest {
  public static void main(String[] args) throws Exception {
    SynonymMap map = new SynonymMap(new FileInputStream("prolog/wn_s.pl"));
    for (String a : args) {
      System.out.printf("%s%n", a);
      for (String s : map.getSynonyms(a)) {
        System.out.printf("%s : %s%n", a, s);
      }
    }
  }
}

$ javac -classpath /usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-2.2.0/contrib/memory/lucene-memory-2.2.0.jar \
  SynonymMapTest.java
$ java -classpath .:/usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-2.2.0/contrib/memory/lucene-memory-2.2.0.jar \
  SynonymMapTest stairway to heaven
stairway
stairway : staircase
to
heaven
heaven : eden
heaven : nirvana
heaven : paradise
$

$ sudo yum install tcl-devel
$ sudo yum install tk-devel
$ cd ~/wordnet
$ wget http://wordnet.princeton.edu/3.0/WordNet-3.0.tar.gz
$ tar zxvpf WordNet-3.0.tar.gz
$ cd WordNet-3.0
$ ./configure
$ make
$ sudo make install
$ /usr/local/WordNet-3.0/bin/wn heaven -synsn

Synonyms/Hypernyms (Ordered by Estimated Frequency) of noun heaven

2 senses of heaven                                                      

Sense 1
Eden, paradise, nirvana, heaven, promised land, Shangri-la
       => region, part

Sense 2
Heaven
       => imaginary place, mythical place, fictitious place
$

$ cd
$ mkdir wordnet
$ cd wordnet
$ wget http://wordnet.princeton.edu/3.0/WNprolog-3.0.tar.gz
$ tar zxvpf WNprolog-3.0.tar.gz
$ java -classpath /usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-2.2.0/contrib/wordnet/lucene-wordnet-2.2.0.jar \
  org.apache.lucene.wordnet.Syns2Index prolog/wn_s.pl index0
Opening Prolog file prolog/wn_s.pl
[1/2] Parsing prolog/wn_s.pl
        2 s(100001740,1,'entity',n,1,11). 0 0 ndecent=0
        4 s(100002137,1,'abstraction',n,6,0). 1 1 ndecent=1
...
        131072 s(113660337,1,'nautical mile',n,2,0). 49957 51928 ndecent=58050
[2/2] Building index to store synonyms,  map sizes are 77651 and 90406
        row=1/77651 doc= Document<stored/uncompressed<syn:between> stored/uncompressed,indexed<word:>>
        row=2/77651 doc= Document<stored/uncompressed<syn:adenine> stored/uncompressed<syn:amp> stored/uncompressed<syn:ampere> \
          stored/uncompressed<syn:angstrom> stored/uncompressed<syn:axerophthol> stored/uncompressed,indexed>
...
        row=32768/77651 doc= Document<stored/uncompressed<syn:redeeming> stored/uncompressed<syn:redemptional> \
          stored/uncompressed<syn:redemptory> stored/uncompressed<syn:saving> stored/uncompressed,indexed<word:redemptive>>
Optimizing..
$

$ java -classpath /usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-2.2.0/contrib/wordnet/lucene-wordnet-2.2.0.jar \
  org.apache.lucene.wordnet.SynExpand index0 'stairway to heaven'
Query: stairway staircase^0.9 heaven eden^0.9 nirvana^0.9 paradise^0.9
$

$ java -classpath /usr/java/lucene-2.2.0/lucene-core-2.2.0.jar:/usr/java/lucene-2.2.0/contrib/wordnet/lucene-wordnet-2.2.0.jar \
  org.apache.lucene.wordnet.SynLookup index0 paradise
Synonyms found for "paradise":
eden
heaven
nirvana
$

$ cd /tmp
$ wget http://dist.codehaus.org/jruby/jruby-bin-1.0.1.tar.gz
$ cd /usr/java
$ tar xvpf /tmp/jruby-bin-1.0.1.tar.gz
$ ln -s jruby-1.0.1 jruby
$ export JAVA_HOME=/usr/java/jdk
$ export JRUBY_HOME=/usr/java/jruby
$ export PATH=$PATH:/$JAVA_HOME/bin:$JRUBY_HOME/bin
$

$ cd ~/samegame
$ jruby samegame.rb
52432214111513151444
23454131532512143554
44245352141115145223
52523312143311335533
52452254245324414354
24521442532114413425
53433111453454233214
13331151323152455423
45251554525341152143
45442232232551441331
54533152252132254212
32524132321413323513
#0 51 actions possible.
### 146 (#2)
#0 [9,3] 2 2 2 1 1
524322141_1513151444
234541315_2512143554
44245352111115145223
52523312133311335533
52452254245324414354
24521442542114413425
53433111443454233214
13331151333152455423
45251554555341152143
45442232232551441331
54533152252132254212
32524132321413323513
#1 49 actions possible.
...
__3______
__5_1____
__1_5____
513432355
#59 1 actions possible.
### 1000 (#59)
#59 [7,0] 5 2 228 1 1000
__3____
__5_1__
__1_5__
5134323
##60 turns, 12 remains, 228 erased, 1000 points.
$

Shell Script 2行 - 3時間
Java 5行 - 10時間

まず最も重要な前提知識として、以下の 3 つのことを遵守することが必須である。
1. 努力しないこと
2. 論理的に考えないこと
3. 頭を使わないこと

<form name="f" action="s" onsubmit="qs(this);return false;">
  <input name="q" type="text">
</form>

<script>
  function qs(tf) {
    var qv = tf.q.value;
    if (window.encodeURIComponent) {
      qv = encodeURIComponent(qv);
      tf.q.value = qv;
    }
    tf.action = "s?q=" + qv ;
    tf.submit();
  }
</script>

$ cd
$ mkdir kazmuzikblog
$ cd kazmuzikblog
$ mkdir src classes lib
$ mv /usr/local/nutch/LiveJournal*.java /usr/local/nutch/RCS src
$ mv /usr/local/nutch/kazmuzikblog .
$

import java.io.Writer;
import java.io.OutputStreamWriter;
import java.io.OutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.sql.SQLException;

public final class LiveJournalHtmlCreator {
  private LiveJournalEntryDatabase db;

  public LiveJournalHtmlCreator() throws SQLException, ClassNotFoundException {
    db = new LiveJournalEntryDatabase();
  }
  public void close() throws SQLException {
    db.close();
  }

  public void printHtml(Writer writer) throws IOException, SQLException {
    LiveJournalEntryHtmlWriter out = new LiveJournalEntryHtmlWriter(writer);
    out.println("<html><head>");
    out.println("<meta http-equiv=\"Content-Type\""
              + " content=\"text/html; charset=utf-8\">");
    out.println("</head><body>");
    out.println("<table border=0 cellspacing=0 cellpadding=8>");
    for (int id : db.getAllEntryIds()) {
      LiveJournalEntry entry = db.select(id);
      out.println("<tr><td bgcolor=#ff0000>");
      out.printf("<a name=\"item%d\"></a>", id);
      out.printEntry(entry);
      out.println("</td></tr>");
    }
    out.println("</table>");
    out.println("</body></html>");
    out.flush();
  }

  public static void main(String args[]) throws Exception {
    OutputStream os = System.out;
    if (args.length > 0) {
      os = new FileOutputStream(args[0]);
    }
    Writer out = new OutputStreamWriter(os, "UTF-8");
    LiveJournalHtmlCreator creator = new LiveJournalHtmlCreator();
    creator.printHtml(out);
    creator.close();
    out.close();
  }
}

<project name="kazmuzikblog" default="compile">
  <property name="src" value="src" />
  <property name="build" value="classes" />
  <property name="lib" value="lib" />
  <property name="nutchPath" value="/usr/local/nutch-0.9" />
  <property name="derbyJar" value="/usr/java/db-derby-10.2.2.0-bin/lib/derby.jar" />

  <target name="compile">
    <javac srcdir="${src}"
           destdir="${build}"
           includes="*.java">
      <classpath>
        <pathelement path="${nutchPath}/nutch-0.9.jar" />
        <pathelement path="${nutchPath}/lib/hadoop-0.12.2-core.jar" />
      </classpath>
    </javac>
  </target>

  <target name="jar" depends="compile">
    <jar destfile="${lib}/kazmuzikblog.jar"
         basedir="${build}"
    />
  </target>

  <target name="html" depends="jar">
    <java classname="LiveJournalHtmlCreator">
      <classpath>
        <pathelement path="${lib}/kazmuzikblog.jar" />
        <pathelement path="${derbyJar}" />
      </classpath>
      <arg value="myjournal.html" />
    </java>
  </target>
</project>

$ ant html
Buildfile: build.xml

compile:
    [javac] Compiling 13 source files to /home/kaz/kazmuzikblog/classes
    [javac] Note: Some input files use or override a deprecated API.
    [javac] Note: Recompile with -Xlint:deprecation for details.

jar:
      [jar] Building jar: /home/kaz/kazmuzikblog/lib/kazmuzikblog.jar

html:

BUILD SUCCESSFUL
Total time: 9 seconds
$ ls -l myjournal.html
-rw-r--r--+ 1 kaz   kaz  1541867 2007-07-04 13:09 myjournal.html
$ 

...
public class LiveJournalEntryDatabase {
  ...
  private static final String sqlSelectAllEntryIdsForTag
    = "select a.id from entryTagTable a, entryTable b"
    + " where a.tag = ? and a.id = b.id order by b.date";
  private static final String sqlSelectAllTags
    = "select distinct tag from entryTagTable order by tag";
  ...
  private PreparedStatement psSelectAllEntryIdsForTag = null;
  private PreparedStatement psSelectAllTags = null;
  ...

  private void prepareStatements() throws SQLException {
    ...
    psSelectAllEntryIdsForTag = conn.prepareStatement(sqlSelectAllEntryIdsForTag);
    psSelectAllTags = conn.prepareStatement(sqlSelectAllTags);
  }
  ...
  public int[] getAllEntryIdsForTag(String tag) throws SQLException {
    psSelectAllEntryIdsForTag.setString(1, tag);
    ResultSet rs = psSelectAllEntryIdsForTag.executeQuery();
    List<Integer> list = new LinkedList<Integer>();
    while (rs.next()) {
      list.add( rs.getInt(1) );
    }
    int n = list.size();
    int[] ids = new int[n];
    for (int i = 0; i < n; i ++) {
      ids[i] = list.get(i);
    }
    return ids;
  }

  public String[] getAllTags() throws SQLException {
    ResultSet rs = psSelectAllTags.executeQuery();
    List<String> list = new LinkedList<String>();
    while (rs.next()) {
      list.add( rs.getString(1) );
    }
    int n = list.size();
    String[] tags = new String[n];
    list.toArray(tags);
    return tags;
  }
  ...
}

...
public final class LiveJournalHtmlCreator {
  ...
  public void printHtml(Writer writer) throws IOException, SQLException {
    printHtml(writer, null);
  }

  public void printHtml(Writer writer, String tag)
      throws IOException, SQLException {
    LiveJournalEntryHtmlWriter out = new LiveJournalEntryHtmlWriter(writer);
    out.println("<html><head>");
    out.println("<meta http-equiv=\"Content-Type\""
              + " content=\"text/html; charset=utf-8\">");
    out.print("<title>Kaz Muzik Blog");
    if (tag != null) {
      out.print(" - Tag: " + tag);
    }
    out.println("</title>");
    out.println("</head><body>");
    out.println("<table border=0 cellspacing=0 cellpadding=8>");
    int[] ids = null;
    if (tag == null) {
      ids = db.getAllEntryIds();
    }
    else {
      ids = db.getAllEntryIdsForTag(tag);
    }
    for (int id : ids) {
      ...
    }
    ...
  }

  public String[] getAllTags() throws SQLException {
    return db.getAllTags();
  }

  public void createFile(String filename, String tag)
      throws SQLException, IOException {
    Writer out
      = new OutputStreamWriter(new FileOutputStream(filename), "UTF-8");
    printHtml(out, tag);
    out.close();
  }

  public static void main(String args[]) throws Exception {
    LiveJournalHtmlCreator creator = new LiveJournalHtmlCreator();
    if (args.length == 0) {
      String filename = "kazmuzikblog-all.html";
      System.err.println("all tags");
      creator.createFile(filename, null);
      for (String tag : creator.getAllTags()) {
        filename = "kazmuzikblog-tag-" + tag + ".html";
        System.err.println("tag=" + tag);
        creator.createFile(filename, tag);
      }
    }
    else {
      String filename = null;
      String tag = null;
      if (args.length > 1) {
        tag = args[0];
        filename = args[1];
      }
      else if (args.length > 0) {
        // tag = null; // ALL
        filename = args[0];
      }
      System.err.println("tag=" + tag);
      creator.createFile(filename, tag);
    }
    creator.close();
  }
}

java -classpath lib/kazmuzikblog.jar:/usr/java/derby/lib/derby.jar LiveJournalFetchPreparator
http://kazuomik.livejournal.com/
104752*I 104659*I 104291*I 104057*I 103902*I 103524 103339 103042 102798 102532
102380 102105 84281 101733 101616 101234 100894 100794 100492 100289
http://kazuomik.livejournal.com/?skip=20
100095 99615 99339 99121 98902 98640 98325 98252 97974 97662
97485 97174 96871 96740 96298 96143 95895 95524 95278 95122
$ java -classpath lib/kazmuzikblog.jar:/usr/java/derby/lib/derby.jar LiveJournalHtmlCreator
all tags
tag=computer technology
tag=music
tag=music and computer
tag=music gear
tag=music technology
tag=programming
tag=test
tag=useful link
tag=アメリカでの転職
tag=アメリカ生活
tag=日本語
$ ls -l kazmuzikblog-*.html 
-rw-r--r--+ 1 kaz  kaz 1555643 2007-07-04 19:08 kazmuzikblog-all.html
-rw-r--r--+ 1 kaz  kaz  874312 2007-07-04 19:08 kazmuzikblog-tag-computer technology.html
-rw-r--r--+ 1 kaz  kaz  270594 2007-07-04 19:08 kazmuzikblog-tag-music and computer.html
-rw-r--r--+ 1 kaz  kaz   62135 2007-07-04 19:08 kazmuzikblog-tag-music gear.html
-rw-r--r--+ 1 kaz  kaz  145081 2007-07-04 19:08 kazmuzikblog-tag-music technology.html
-rw-r--r--+ 1 kaz  kaz    1950 2007-07-04 19:08 kazmuzikblog-tag-music.html
-rw-r--r--+ 1 kaz  kaz  778115 2007-07-04 19:08 kazmuzikblog-tag-programming.html
-rw-r--r--+ 1 kaz  kaz    3524 2007-07-04 19:08 kazmuzikblog-tag-test.html
-rw-r--r--+ 1 kaz  kaz    8946 2007-07-04 19:08 kazmuzikblog-tag-useful link.html
-rw-r--r--+ 1 kaz  kaz   46437 2007-07-04 19:08 kazmuzikblog-tag-アメリカでの転職.html
-rw-r--r--+ 1 kaz  kaz  237594 2007-07-04 19:08 kazmuzikblog-tag-アメリカ生活.html
-rw-r--r--+ 1 kaz  kaz    1883 2007-07-04 19:08 kazmuzikblog-tag-日本語.html
$

$ java -classpath .:/usr/java/derby/lib/derby.jar LiveJournalFetchPreparator -init
http://kazuomik.livejournal.com/
103524*I 103339*I 103042*I 102798*I 102532*I 102380*I 102105*I 84281*I 101733*I 101616*I
101234*I 100894*I 100794*I 100492*I 100289*I 100095*I 99615*I 99339*I 99121*I 98902*I
http://kazuomik.livejournal.com/?skip=20
98640*I 98325*I 98252*I 97974*I 97662*I 97485*I 97174*I 96871*I 96740*I 96298*I
...
l8640*I 8328*I 8982*I 8161*I 7327*I 7694*I 7456*I 6946*I 6753*I 6600*I
http://kazuomik.livejournal.com/?skip=380
5941 6278 5653 5503 5242 5085 4837 3672 3518 3209
2876 495 722 1960 4044 1598 4195 1332 4582 1120
$ wc -l urls-kazmuzikblog/nutch
400 urls-kazmuzikblog/nutch
$

$ bin/nutch inject tmp-crawldb urls-kazmuzikblog
...
$ bin/nutch generate tmp-crawldb tmp-segments
...
Generator: segment: tmp-segments/20070703074227
...
$ bin/nutch fetch tmp-segments/20070703074227
Fetcher: starting
Fetcher: segment: tmp-segments/20070703074227
Fetcher: threads: 10
fetching http://kazuomik.livejournal.com/88261.html
...
fetching http://kazuomik.livejournal.com/61706.html
fetch of http://kazuomik.livejournal.com/61706.html failed with:\
 java.net.SocketTimeoutException: Read timed out
fetching http://kazuomik.livejournal.com/103524.html
...
fetching http://kazuomik.livejournal.com/81466.html
Fetcher: done
$

$ bin/nutch readseg -list -dir tmp-segments
NAME            GENERATED  FETCHER START        FETCHER END          FETCHED PARSED
20070703074227  400        2007-07-03T07:43:29  2007-07-03T08:18:01  400     346
$ 

...
public SimpleTranslator {
 ...

  public String translate(String enText) {
    if (enText == null || enText.length() < 1) {
      return "";
    }
    //
    String jaText = dict.get(enText);
    if (jaText != null) {
      return jaText;
    }
    //
    int enTextLen = enText.length();
    if (enText.charAt(enTextLen-1) == '.') {
      return translate(enText.substring(0,enTextLen-1)) + "。";
    }
    //
    String[] enWords = enText.split(" ");
    int len = enWords.length;
    if (len < 2) {
      return enText;
    }
    //
    int m = enText.indexOf(" is ");
    if (m > 0) {
      return concatJapaneseTerms(
        translate( enText.substring(0,m) ),
        "は、",
        translate( enText.substring(m+4) ),
        "です" );
    }
    //
    ....
  }
  ...
  public static void main(String[] args) throws Exception {
    ...
    String jaTerm = translator.translate(enTerm.toLowerCase());
    ...
  }
}

$ cat dict7.txt
this = これ
that = あれ
pen = ペン
notebook = ノート
desk = 机
the = 
a = 
an = 
$ cat en_text7.txt
This is a pen.
That is a desk.
$ cat en_text7.txt | java SimpleTranslator dict7.txt
This is a pen. = これは、ペンです。
That is a desk. = あれは、机です。
$ 

import java.util.Map;
import java.util.HashMap;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.FileInputStream;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;

public class SimpleTranslator {
  private static Map<String,String> dictReverse;
  static {
    dictReverse = new HashMap<String,String>();
    dictReverse.put("to", "への");
    dictReverse.put("of", "の");
    dictReverse.put("on", "の上の");
  }

  private static final int MAX_WORD_NUMBER = 8;
  private Map<String,String> dict;
  private Map<String,String>[] dicts;

  public SimpleTranslator(String filename) throws IOException {
    createDictionaries();
    readDictionary(filename);
    enhanceDictionary();
  }

  private void createDictionaries() {
    dict = new HashMap<String,String>();
    dicts = new Map[MAX_WORD_NUMBER + 1];
    for (int i = 0; i < dicts.length; i++) {
      dicts[i] = new HashMap<String,String>();
    }
  }


  private void readDictionary(String filename) throws IOException {
    BufferedReader in = new BufferedReader( new InputStreamReader(
                          new FileInputStream(filename), "UTF-8") );
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      String[] terms = line.split(" = ");
      if (terms.length < 2 || terms[1] == null) {
        putTerm(terms[0], "");
      }
      else {
        putTerm(terms[0], terms[1]);
      }
    }
    in.close();
  }

  private void putTerm(String enTerm, String jaTerm) {
    dict.put(enTerm, jaTerm);
    String[] words = enTerm.split(" ");
    if (words.length > MAX_WORD_NUMBER) {
      dicts[0].put(enTerm, jaTerm);
    }
    else {
      dicts[words.length].put(enTerm, jaTerm);
    }
  }

  private int enhanceDictionary() {
    int count = 0;
    while (true) {
      int n = enhanceDictionary1();
      if (n < 1) {
        break;
      }
      count += n;
      System.err.printf("(%d entries added to the dictionary)%n", n);
    }
    return count;
  }

  private int enhanceDictionary1() {
    Map<String,String> map = new HashMap<String,String>();
    for (String enTerm : dict.keySet()) {
      String[] enWords = enTerm.split(" ");
      if (enWords.length < 2) {
        continue;
      }
      String jaTerm = dict.get(enTerm);
      String[] jaWords = new String[enWords.length];
      int index = -1;
      for (int i = 0; i < enWords.length; i++) {
        String jaWord = dicts[1].get(enWords[i]);
        if (jaWord == null) {
          index = i;
          break;
        }
        if (! jaTerm.startsWith(jaWord)) {
          break;
        }
        jaTerm = jaTerm.substring(jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      for (int i = enWords.length - 1; i > index; i--) {
        String jaWord = dicts[1].get(enWords[i]);
        if (jaWord == null) {
          index = -1;
          break;
        }
        if (! jaTerm.endsWith(jaWord)) {
          index = -1;
          break;
        }
        jaTerm = jaTerm.substring(0, jaTerm.length() - jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      if (jaTerm.length() < 1) {
        continue;
      }
      if (dicts[1].get(enWords[index]) != null) {
        continue;
      }
      String oldJaTerm = map.get(enWords[index]);
      if (oldJaTerm == null) {
        map.put(enWords[index], jaTerm);
      }
      else if (oldJaTerm.length() < jaTerm.length()) {
        map.put(enWords[index], jaTerm);
      }
    }
    for (String enTerm : map.keySet()) {
      String jaTerm = map.get(enTerm);
      putTerm(enTerm, jaTerm);
      System.err.printf("* %s = %s%n", enTerm, jaTerm);
    }
    return map.size();
  }

  public String translate(String enText) {
    String jaText = dict.get(enText);
    if (jaText != null) {
      return jaText;
    }
    //
    String[] enWords = enText.split(" ");
    int len = enWords.length;
    if (len < 2) {
      return enText;
    }
    //
    for (String enWord : dictReverse.keySet()) {
      int n = enText.indexOf(" "+ enWord + " ");
      if (n < 0) {
        continue;
      }
      String jaTerm = dictReverse.get(enWord);
      if (jaTerm != null) {
        return concatJapaneseTerms(
          translate( enText.substring(n + 2 + enWord.length()) ),
          jaTerm,
          translate( enText.substring(0, n) ) );
      }
    }
    //
    if (--len > MAX_WORD_NUMBER) {
      len = MAX_WORD_NUMBER;
    }
    while (len > 0) {
      for (int i = 0; i + len <= enWords.length; i++) {
        String enTerm = concatEnglishWords(enWords,i,len);
        String jaTerm = dicts[len].get( enTerm );
        if (jaTerm != null) {
          return concatJapaneseTerms(
            translate(concatEnglishWords(enWords,0,i)),
            jaTerm,
            translate(concatEnglishWords(enWords,i+1, enWords.length-len-i)) );
        }
      }
      len --;
    }
    return enText;
  }

  private String concatEnglishWords(String[] words, int offset, int len) {
    if (words== null || len < 1) {
      return "";
    }
    len += offset;
    if (len > words.length) {
      len = words.length;
    }
    String prevWord = null;
    StringBuilder sb = new StringBuilder();
    for (int i = offset; i < len; i++) {
      String word = words[i];
      if (word == null || word.length() < 1) {
        continue;
      }
      if (prevWord != null) {
        sb.append(" ");
      }
      sb.append(word);
      prevWord = word;
    }
    return sb.toString();
  }

  private String concatJapaneseTerms(String... words) {
    if (words == null || words.length < 1) {
      return "";
    }
    //
    String prevWord = null;
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < words.length; i++) {
      String word= words[i];
      if (word == null || word.length() < 1) {
        continue;
      }
      if (prevWord != null) {
        int prevWordLen = prevWord.length();
        if (prevWord.charAt(prevWordLen-1) < '\u0100') {
          sb.append(" ");
        }
        else if (word.charAt(0) < '\u0100') {
          sb.append(" ");
        }
      }
      sb.append(word);
      prevWord= word;
    }
    return sb.toString();
  }

  public static void main(String[] args) throws Exception {
    BufferedReader in = new BufferedReader(
                          new InputStreamReader(System.in, "UTF-8"));
    PrintWriter out = new PrintWriter(
                          new OutputStreamWriter(System.out, "UTF-8"));
    SimpleTranslator translator = new SimpleTranslator(args[0]);
    while (true) {
      String enTerm = in.readLine();
      if (enTerm == null) {
        break;
      }
      String jaTerm = translator.translate(enTerm);
      out.printf("%s = %s%n", enTerm, jaTerm);
    }
    out.flush();
    out.close();
    in.close();
  }
}

$ cat dict6.txt
stairway = 階段
heaven = 天国
one = ひとつ
them = それら
notebook = ノート
desk = 机
the = 
a = 
an = 
$ cat en_text6.txt
the stairway to heaven
one of them
a notebook on the desk
$ cat en_text6.txt | java SimpleTranslator dict6.txt
the stairway to heaven = 天国への階段
one of them = それらのひとつ
a notebook on the desk = 机の上のノート
$

$ cat dict5.txt
stairway = 階段
heaven = 天国
$ cat en_text5.txt
stairway to heaven
$ cat en_text5.txt | java SimpleTranslator dict5.txt
stairway to heaven = 天国への階段
$

...
public class SimpleTranslator {
  ...

  public String translate(String enText) {
    String jaText = dict.get(enText);
    if (jaText != null) {
      return jaText;
    }
    //
    String[] enWords = enText.split(" ");
    int len = enWords.length;
    if (len < 2) {
      return enText;
    }
    //
    int n = enText.indexOf(" to ");
    if (n > 0) {
      StringBuilder sb = new StringBuilder();
      String enTerm = enText.substring(n + 4);
      String jaTerm = translate(enTerm);
      sb.append(jaTerm);
      if (jaTerm.charAt(jaTerm.length()-1) < '\u0100') {
        sb.append(" ");
      }
      sb.append("\u3078\u306e"); // への (to)
      enTerm = enText.substring(0, n);
      jaTerm = translate(enTerm);
      if (jaTerm.charAt(0) < '\u0100') {
        sb.append(" ");
      }
      sb.append(jaTerm);
      return sb.toString();
    }
    //
    if (--len > MAX_WORD_NUMBER) {
      len = MAX_WORD_NUMBER;
    }
    while (len > 0) {
      for (int i = 0; i + len <= enWords.length; i++) {
        String enTerm = concatWords(enWords,i,len);
        String jaTerm = dicts[len].get( enTerm );
        if (jaTerm != null) {
          StringBuilder sb = new StringBuilder();
          if (i > 0) {
            String jaTerm1 = translate( concatWords(enWords,0,i) );
            sb.append(jaTerm1);
            if (jaTerm1.charAt(jaTerm1.length()-1) < '\u0100') {
              sb.append(" ");
            }
          }
          sb.append(jaTerm);
          if (i +len < enWords.length) {
            String jaTerm2 = translate( concatWords(enWords,i+1, enWords.length-i-1) );
            if (jaTerm2.charAt(0) < '\u0100') {
              sb.append(" ");
            }
            sb.append(jaTerm2);
          }
          return sb.toString();
        }
      }
      len --;
    }
    return enText;
  }

  ...
}

pulibc class SimpleTranslator {
  ...

  private int enhanceDictionary1() {
    Map<String,String> map = new HashMap<String,String>();
    for (String enTerm : dict.keySet()) {
      String[] enWords = enTerm.split(" ");
      if (enWords.length < 2) {
        continue;
      }
      String jaTerm = dict.get(enTerm);
      String[] jaWords = new String[enWords.length];
      int index = -1;
      for (int i = 0; i < enWords.length; i++) {
        String jaWord = dicts[1].get(enWords[i]);
        if (jaWord == null) {
          index = i;
          break;
        }
        if (! jaTerm.startsWith(jaWord)) {
          break;
        }
        jaTerm = jaTerm.substring(jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      for (int i = enWords.length - 1; i > index; i--) {
        String jaWord = dicts[1].get(enWords[i]);
        if (jaWord == null) {
          index = -1;
          break;
        }
        if (! jaTerm.endsWith(jaWord)) {
          index = -1;
          break;
        }
        jaTerm = jaTerm.substring(0, jaTerm.length() - jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      if (jaTerm.length() < 1) {
        continue;
      }
      if (dicts[1].get(enWords[index]) != null) {
        continue;
      }
      String oldJaTerm = map.get(enWords[index]);
      if (oldJaTerm == null) {
        map.put(enWords[index], jaTerm);
      }
      else if (oldJaTerm.length() < jaTerm.length()) {
        map.put(enWords[index], jaTerm);
      }
    }
    for (String enTerm : map.keySet()) {
      String jaTerm = map.get(enTerm);
      dict.put(enTerm, jaTerm);
      dicts[1].put(enTerm, jaTerm);
      System.err.printf("* %s = %s%n", enTerm, jaTerm);
    }
    return map.size();
  }

  ...
}

$ cat dict4.txt
black = 黒い
house = 家
dog = 犬
house dog = 番犬
big dog = 大きな犬
$ cat en_text4.txt
big house dog
$ javac SimpleTranslator.java
$ cat en_text4.txt | java SimpleTranslator dict4.txt
* big = 大きな
(1 entries added to the dictionary)
big house dog = 大きな番犬
$ 

$ cat dict3.txt
black = 黒い
house = 家
dog = 犬
house dog = 番犬
$ cat en_text3.txt
black house dog
$ cat en_text3.txt | java SimpleTranslator dict3.txt
black house dog = 黒い家犬
$

...
public class SimpleTranslator {
  private static final int MAX_WORD_NUMBER = 8;
  private Map<String,String> dict;
  private Map<String,String>[] dicts;

  public SimpleTranslator(String filename) throws IOException {
    readDictionary(filename);
    enhanceDictionary();
  }

  private void readDictionary(String filename) throws IOException {
    dict = new HashMap<String,String>();
    dicts = new Map[MAX_WORD_NUMBER + 1];
    for (int i = 0; i < dicts.length; i++) {
      dicts[i] = new HashMap<String,String>();
    }
    //
    BufferedReader in = new BufferedReader(
                          new InputStreamReader(
                            new FileInputStream(filename), "UTF-8"));
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      String[] terms = line.split(" = ");
      dict.put(terms[0], terms[1]);
      String[] words = terms[0].split(" ");
      if (words.length > MAX_WORD_NUMBER) {
        dicts[0].put(terms[0], terms[1]);
      }
      else {
        dicts[words.length].put(terms[0], terms[1]);
      }
    }
    in.close();
  }

  ...

  public String translate(String enText) {
    String jaText = dict.get(enText);
    if (jaText != null) {
      return jaText;
    }
    //
    String[] enWords = enText.split(" ");
    int len = enWords.length;
    if (len < 2) {
      return enText;
    }
    //
    if (--len > MAX_WORD_NUMBER) {
      len = MAX_WORD_NUMBER;
    }
    while (len > 0) {
      for (int i = 0; i + len <= enWords.length; i++) {
        String enTerm = concatWords(enWords,i,len);
        String jaTerm = dicts[len].get( enTerm );
        if (jaTerm != null) {
          StringBuilder sb = new StringBuilder();
          if (i > 0) {
            String jaTerm1 = translate( concatWords(enWords,0,i) );
            sb.append(jaTerm1);
            if (jaTerm1.charAt(jaTerm1.length()-1) < '\u0100') {
              sb.append(" ");
            }
          }
          sb.append(jaTerm);
          if (i +len < enWords.length) {
            String jaTerm2
              = translate( concatWords(enWords,i+1, enWords.length-i-1) );
            if (jaTerm2.charAt(0) < '\u0100') {
              sb.append(" ");
            }
            sb.append(jaTerm2);
          }
          return sb.toString();
        }
      }
      len --;
    }
    return enText;
  }

  private String concatWords(String[] words, int offset, int len) {
    if (len < 1) {
      return "";
    }
    StringBuilder sb = new StringBuilder(words[offset]);
    for (int i = 1; i < len; i++) {
      sb.append(" ");
      sb.append(words[offset + i]);
    }
    return sb.toString();
  }

  ...
}

$ javac SimpleTranslator.java
$ cat en_text3.txt | java SimpleTranslator dict3.txt
black house dog = 黒い番犬
$

...
public class SimpleTranslator {
  ...

  public SimpleTranslator(String filename) throws IOException {
    readDictionary(filename);
    enhanceDictionary();
  }

  ...

  private int enhanceDictionary() {
    int count = 0;
    while (true) {
      int n = enhanceDictionary1();
      if (n < 1) {
        break;
      }
      count += n;
      System.err.printf("(%d entries added to the dictionary)%n", n);
    }
    return count;
  }

  private int enhanceDictionary1() {
    Map<String,String> map = new HashMap<String,String>();
    for (String enTerm : dict.keySet()) {
      String[] enWords = enTerm.split(" ");
      if (enWords.length < 2) {
        continue;
      }
      String jaTerm = dict.get(enTerm);
      String[] jaWords = new String[enWords.length];
      int index = -1;
      for (int i = 0; i < enWords.length; i++) {
        String jaWord = dict.get(enWords[i]);
        if (jaWord == null) {
          index = i;
          break;
        }
        if (! jaTerm.startsWith(jaWord)) {
          break;
        }
        jaTerm = jaTerm.substring(jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      for (int i = enWords.length - 1; i > index; i--) {
        String jaWord = dict.get(enWords[i]);
        if (jaWord == null) {
          index = -1;
          break;
        }
        if (! jaTerm.endsWith(jaWord)) {
          index = -1;
          break;
        }
        jaTerm = jaTerm.substring(0, jaTerm.length() - jaWord.length());
        continue;
      }
      if (index < 0) {
        continue;
      }
      if (jaTerm.length() < 1) {
        continue;
      }
      if (dict.get(enWords[index]) != null) {
        continue;
      }
      String oldJaTerm = map.get(enWords[index]);
      if (oldJaTerm == null) {
        map.put(enWords[index], jaTerm);
      }
      else if (oldJaTerm.length() < jaTerm.length()) {
        map.put(enWords[index], jaTerm);
      }
    }
    for (String enTerm : map.keySet()) {
      String jaTerm = map.get(enTerm);
      dict.put(enTerm, jaTerm);
      System.err.printf("* %s = %s%n", enTerm, jaTerm);
    }
    return map.size();
  }

  ...
}

$ javac SimpleTranslator.java
$ cat dict2.txt
dog = 犬
black dog = 黒い犬
white dog = 白い犬
black cat = 黒い猫
$ cat en_text2.txt
white cat
$ cat en_text2.txt | java SimpleTranslator dict2.txt
* white = 白い
* black = 黒い
(2 entries added to the dictionary)
* cat = 猫
(1 entries added to the dictionary)
white cat = 白い猫
$ 

import java.util.Map;
import java.util.HashMap;
import java.io.IOException;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.FileInputStream;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;

public class SimpleTranslator {
  private Map<String,String> dict;

  public SimpleTranslator(String filename) throws IOException {
    readDictionary(filename);
  }

  private void readDictionary(String filename) throws IOException {
    dict = new HashMap<String,String>();
    BufferedReader in = new BufferedReader(
                          new InputStreamReader(
                            new FileInputStream(filename), "UTF-8"));
    while (true) {
      String line = in.readLine();
      if (line == null) {
        break;
      }
      String[] terms = line.split(" = ");
      dict.put(terms[0], terms[1]);
    }
    in.close();
  }

  public String translate(String enText) {
    String jaText = dict.get(enText);
    if (jaText != null) {
      return jaText;
    }
    //
    StringBuilder sb = new StringBuilder();
    boolean translated = true;
    boolean first = true;
    for (String enWord : enText.split(" ")) {
      String jaWord = dict.get(enWord);
      if (jaWord == null) {
        if (! first) {
          sb.append(" ");
        }
        sb.append(enWord);
        translated = false;
      }
      else {
        if (! translated) {
          sb.append(" ");
        }
        sb.append(jaWord);
        translated = true;
      }
      first = false;
    }
    return sb.toString();
  }

  public static void main(String[] args) throws Exception {
    BufferedReader in = new BufferedReader(
                          new InputStreamReader(System.in, "UTF-8"));
    PrintWriter out = new PrintWriter(
                          new OutputStreamWriter(System.out, "UTF-8"));
    SimpleTranslator translator = new SimpleTranslator(args[0]);
    while (true) {
      String enTerm = in.readLine();
      if (enTerm == null) {
        break;
      }
      String jaTerm = translator.translate(enTerm);
      out.printf("%s = %s%n", enTerm, jaTerm);
    }
    out.flush();
    out.close();
    in.close();
  }
}

$ javac SimpleTranslator.java
$ cat dict1.txt
black = 黒い
dog = 犬
$ cat en_text1.txt
black dog
white dog
black cat
white cat
$ cat en_text1.txt | java SimpleTranslator dict1.txt
black dog = 黒い犬
white dog = white 犬
black cat = 黒い cat
white cat = white cat
$

import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.FileOutputStream;
import java.util.List;

public class LiveJournalFetchPreparator {
  public static void main(String[] args) throws Exception {
    boolean init = false;
    if (args.length > 0 && args[0].equals("-init")) {
      init = true;
    }
    LiveJournalDatabaseManager manager = new LiveJournalDatabaseManager(init);
    List list = manager.updateEntries();
    PrintWriter out
      = new PrintWriter(
          new OutputStreamWriter(
            new FileOutputStream("urls-kazmuzikblog/nutch"), "US-ASCII"));
    for (int id : list) {
      out.printf("http://kazuomik.livejournal.com/%d.html%n", id);
    }
    out.flush();
    out.close();
    //
    out = new PrintWriter(new OutputStreamWriter(
            new FileOutputStream("conf/crawl-urlfilter.txt"), "US-ASCII"));
    out.println("+http://kazuomik.livejournal.com/[1-9][0-9]*.html$");
    out.println("-.");
    out.flush();
    out.close();
  }
}

$ java -classpath .:/usr/java/derby/lib/derby.jar LiveJournalFetchPreparator
http://kazuomik.livejournal.com/
91272*I 91120*I 84281 90712 90539 90319 89953 89809 89537 89171
88875 88669 88513 88261 87862 87654 87455 87177 86869 86541
http://kazuomik.livejournal.com/?skip=20
86459 86194 85837 85750 85372 85193 84912 84571 59760 84106
83846 83562 83109 70808 83425 82794 82561 82243 81991 81756
$ cat urls-kazmuzikblog/nutch
http://kazuomik.livejournal.com/91272.html
http://kazuomik.livejournal.com/91120.html
$ 

import java.net.URL;
import java.net.URLConnection;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.sql.SQLException;
import java.util.List;
import java.util.ArrayList;

public class LiveJournalDatabaseManager extends LiveJournalEntryDatabase {

  private static boolean verbose = true;

  public LiveJournalDatabaseManager() throws SQLException, ClassNotFoundException {
    this(false);
  }

  public LiveJournalDatabaseManager(boolean init) throws SQLException, ClassNotFoundException {
    super(init);
  }

  public List updateEntries(int nSkip) throws SQLException, IOException {
    String urlString = "http://kazuomik.livejournal.com/";
    if (nSkip > 0) {
      urlString += ("?skip=" + nSkip);
    }
    if (verbose) {
      System.err.println(urlString);
    }
    URL url = new URL(urlString);
    URLConnection conn = url.openConnection();
    InputStream in = conn.getInputStream();
    LiveJournalEntryParser parser
      = new LiveJournalEntryParser(new InputStreamReader(in, "UTF-8"));
    List list = new ArrayList(20);
    int count = 0;
    while (true) {
      LiveJournalEntry entry = parser.parse();
      if (entry == null) {
        break;
      }
      if (verbose) {
        System.err.printf("%d", entry.getId());
      }
      int id = entry.getId();
      LiveJournalEntry oldEntry = select(id);
      if (oldEntry == null) {
        insert(entry);
        list.add(id);
        if (verbose) {
          System.err.print("*I");
        }
      }
      else if (! entry.equals(oldEntry)) {
        update(entry);
        list.add(id);
        if (verbose) {
          System.err.print("*U");
        }
      }
      else {
      }
      count ++;
      if (verbose) {
        if (count % 10 == 0) {
          System.err.println();
        }
        else {
          System.err.print(" ");
        }
        System.err.flush();
      }
    }
    if (verbose) {
      if (count % 10 != 0) { 
        System.err.println();
      }
    }
    return list;
  }

  public List updateEntries() throws SQLException, IOException {
    List allList = new ArrayList();
    int n = 0;
    while (true) {
       List list = updateEntries(n);
       if (list.size() == 0)
       {
         break;
       }
       allList.addAll(list);
       n += 20;
       try {
         Thread.sleep(5000);
       }
       catch (InterruptedException e) {
       }
    }
    return allList;
  }
}

...
public class LiveJournalEntry implements Serializable {
  ...

  public boolean equals(Object other) {
    if (! (other instanceof LiveJournalEntry)) {
      return false;
    }
    LiveJournalEntry that = (LiveJournalEntry)other;
    if (id != that.id) {
      return false;
    }
    if (date.getTime() / 60000 != that.date.getTime() / 60000)
    {
      return false;
    }
    if (! title.equals(that.title)) {
      return false;
    }
    if (! entry.equals(that.entry)) {
      return false;
    }
    if (mood == null) {
      if (that.mood != null) {
        return false;
      }
    }
    else if (that.mood == null) {
      return false;
    }
    else if (! mood.equals(that.mood)) {
      return false;
    }
    if (location == null) {
      if (that.location != null) {
        return false;
      }
    }
    else if (that.location == null) {
      return false;
    }
    else if (! location.equals(that.location)) {
      return false;
    }
    if (tags == null) {
      if (that.tags != null) {
        return false;
      }
    }
    else if (that.tags == null) {
      return false;
    }
    else {
      if (tags.length != that.tags.length) {
        return false;
      }
      for (int i = 0 ; i < tags.length; i++) {
        if (! tags[i].equals(that.tags[i])) {
          return false;
        }
      }
    }
    return true;
  }
}

import java.io.OutputStreamWriter;

public final class LiveJournalHtmlCreator {

  public static void main(String argv[]) throws Exception {
    LiveJournalEntryDatabase db = new LiveJournalEntryDatabase();

    LiveJournalEntryHtmlWriter out
      = new LiveJournalEntryHtmlWriter(
          new OutputStreamWriter(System.out, "UTF-8"));
    out.println("<html><body>");
    out.println("<able border=0 cellspacing=0 cellpadding=8>");
    for (int id : db.getAllEntryIds()) {
      LiveJournalEntry entry = db.select(id);
      out.println("<tr><td bgcolor=#ff0000>");
      out.printf("<a name=\"item%d\"></a>", id);
      out.printEntry(entry);
      out.println("</td></tr>");
    }
    out.println("</table>");
    out.println("</body></html>");
    out.close();

    db.close();
  }
}

$ javac LiveJournalHtmlCreator.java
$ java -classpath .:/usr/java/jdk/db/derby.jar LiveJournalHtmlCreator > myjournal3.html
$

...

public class LiveJournalEntryDatabase {

  ...

  private static final String sqlCreateEntryTable
    = "create table entryTable "
    + "(id int not null, date timestamp not null, title varchar(256) not null,"
    + " mood varchar(32), location varchar(32),"
    + " primary key(id))";
  private static final String sqlCreateEntryBodyTable
    = "create table entryBodyTable (id int not null, entry blob,"
    + " primary key(id))";
  private static final String sqlCreateEntryTagTable
    = "create table entryTagTable (id int not null, tag varchar(64))";

  private static final String sqlCreateEntryDateIndex
    = "create index entryDateIndex on entryTable (date asc)";
  private static final String sqlCreateTagIdIndex
    = "create index tagIdIndex on entryTagTable (id asc)";

  ...

  public void createTables() throws SQLException {
    executeSql(sqlCreateEntryTable);
    executeSql(sqlCreateEntryBodyTable);
    executeSql(sqlCreateEntryTagTable);
    executeSql(sqlCreateEntryDateIndex);
    executeSql(sqlCreateTagIdIndex);
  }

  ...
}

import java.net.URL;
import java.net.URLConnection;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.sql.SQLException;

public class LiveJournalDatabaseSetup extends LiveJournalEntryDatabase {
  public LiveJournalDatabaseSetup() throws SQLException, ClassNotFoundException {
    super(true);
  }

  public int insertEntries(int nSkip) throws SQLException, IOException {
    String urlString = "http://kazuomik.livejournal.com/";
    if (nSkip > 0) {
      urlString += ("?skip=" + nSkip);
    }
    System.out.println(urlString);
    URL url = new URL(urlString);
    URLConnection conn = url.openConnection();
    InputStream in = conn.getInputStream();
    LiveJournalEntryParser parser
      = new LiveJournalEntryParser(new InputStreamReader(in, "UTF-8"));
    int count = 0;
    while (true) {
      LiveJournalEntry entry = parser.parse();
      if (entry == null) {
        break;
      }
      System.out.printf("%d", entry.getId());
      System.out.flush();
      insert(entry);
      count ++;
      if (count % 10 == 0) {
        System.out.println();
      }
      else {
        System.out.print(" ");
      }
    }
    if (count % 10 != 0) {
      System.out.println();
    }
    return count;
  }

  public static void main(String[] args) throws Exception {
    int n = 0;
    LiveJournalDatabaseSetup db = new LiveJournalDatabaseSetup();
    while (true) {
       int m = db.insertEntries(n);
       if (m < 20) {
         break;
       }
       n += 20;
       try {
         Thread.sleep(5000);
       }
       catch (InterruptedException e) {
       }
    }
    db.close();
  }
}

$ javac LiveJournalEntryDatabase.java
$ javac LiveJournalDatabaseSetup.java
$ java -classpath .:/usr/java/jdk1.6.0_01/db/derby.jar LiveJournalEntryUpdater
http://kazuomik.livejournal.com/
80810 80540 80298 79912 59760 79813 79495 79224 79017 78745
78462 78252 77846 77703 77432 77261 77050 76685 76427 76062
http://kazuomik.livejournal.com/?skip=20
76024 75731 75404 75135 74998 74600 70808 74410 74000 73761
...
http://kazuomik.livejournal.com/?skip=280 
8776 8640 8328 8982 8161 7327 7694 7456 6946 6753
6600 5941 6278 5653 5503 5242 5085 4837 3672 3518
http://kazuomik.livejournal.com/?skip=300 
3209 2876 495 722 1960 4044 1598 4195 1332 4582 
1120 1008 2409 2080 2603 
$