/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.protocol.ftp;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.Socket;
import java.util.List;
//import java.util.LinkedList;

import org.apache.commons.net.MalformedServerReplyException;
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPCommand;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPFileEntryParser;
import org.apache.commons.net.ftp.FTPReply;
import org.apache.commons.net.ftp.FTPConnectionClosedException;

/***********************************************
 * Client.java encapsulates functionalities necessary for nutch to get dir list
 * and retrieve file from an FTP server. This class takes care of all low level
 * details of interacting with an FTP server and provides a convenient higher
 * level interface.
 * 
 * Modified from FtpClient.java in apache commons-net.
 * 
 * Notes by John Xing: ftp server implementations are hardly uniform and none
 * seems to follow RFCs whole-heartedly. We have no choice, but assume common
 * denominator as following: (1) Use stream mode for data transfer. Block mode
 * will be better for multiple file downloading and partial file downloading.
 * However not every ftpd has block mode support. (2) Use passive mode for data
 * connection. So nutch will work if we run behind firewall. (3) Data connection
 * is opened/closed per ftp command for the reasons listed in (1). There are ftp
 * servers out there, when partial downloading is enforeced by closing data
 * channel socket on our client side, the server side immediately closes control
 * channel (socket). Our codes deal with such a bad behavior. (4) LIST is used
 * to obtain remote file attributes if possible. MDTM & SIZE would be nice, but
 * not as ubiquitously implemented as LIST. (5) Avoid using ABOR in single
 * thread? Do not use it at all.
 * 
 * About exceptions: Some specific exceptions are re-thrown as one of
 * FtpException*.java In fact, each function throws FtpException*.java or pass
 * IOException.
 * 
 * @author John Xing
 ***********************************************/

public class Client extends FTP {
  private int __dataTimeout;
  private int __passivePort;
  private String __passiveHost;
  // private int __fileType, __fileFormat;
  private boolean __remoteVerificationEnabled;
  // private FTPFileEntryParser __entryParser;
  private String __systemName;

  // constructor
  public Client() {
    __initDefaults();
    __dataTimeout = -1;
    __remoteVerificationEnabled = true;
  }

  // defaults when initialize
  private void __initDefaults() {
    __passiveHost = null;
    __passivePort = -1;
    __systemName = null;
    /*
     * __fileType = FTP.ASCII_FILE_TYPE; __fileFormat =
     * FTP.NON_PRINT_TEXT_FORMAT; __entryParser = null;
     */}

  // parse reply for pass()
  private void __parsePassiveModeReply(String reply)
      throws MalformedServerReplyException {
    int i, index, lastIndex;
    String octet1, octet2;
    StringBuffer host;

    reply = reply.substring(reply.indexOf('(') + 1, reply.indexOf(')')).trim();

    host = new StringBuffer(24);
    lastIndex = 0;
    index = reply.indexOf(',');
    host.append(reply.substring(lastIndex, index));

    for (i = 0; i < 3; i++) {
      host.append('.');
      lastIndex = index + 1;
      index = reply.indexOf(',', lastIndex);
      host.append(reply.substring(lastIndex, index));
    }

    lastIndex = index + 1;
    index = reply.indexOf(',', lastIndex);

    octet1 = reply.substring(lastIndex, index);
    octet2 = reply.substring(index + 1);

    // index and lastIndex now used as temporaries
    try {
      index = Integer.parseInt(octet1);
      lastIndex = Integer.parseInt(octet2);
    } catch (NumberFormatException e) {
      throw new MalformedServerReplyException(
          "Could not parse passive host information.\nServer Reply: " + reply);
    }

    index <<= 8;
    index |= lastIndex;

    __passiveHost = host.toString();
    __passivePort = index;
  }

  /**
   * open a passive data connection socket
   * 
   * @param command
   * @param arg
   * @return
   * @throws IOException
   * @throws FtpExceptionCanNotHaveDataConnection
   */
  protected Socket __openPassiveDataConnection(int command, String arg)
      throws IOException, FtpExceptionCanNotHaveDataConnection {
    Socket socket;

    // // 20040317, xing, accommodate ill-behaved servers, see below
    // int port_previous = __passivePort;

    if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
      throw new FtpExceptionCanNotHaveDataConnection("pasv() failed. "
          + getReplyString());

    try {
      __parsePassiveModeReply(getReplyStrings()[0]);
    } catch (MalformedServerReplyException e) {
      throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
    }

    // // 20040317, xing, accommodate ill-behaved servers, see above
    // int count = 0;
    // System.err.println("__passivePort "+__passivePort);
    // System.err.println("port_previous "+port_previous);
    // while (__passivePort == port_previous) {
    // // just quit if too many tries. make it an exception here?
    // if (count++ > 10)
    // return null;
    // // slow down further for each new try
    // Thread.sleep(500*count);
    // if (pasv() != FTPReply.ENTERING_PASSIVE_MODE)
    // throw new FtpExceptionCanNotHaveDataConnection(
    // "pasv() failed. " + getReplyString());
    // //return null;
    // try {
    // __parsePassiveModeReply(getReplyStrings()[0]);
    // } catch (MalformedServerReplyException e) {
    // throw new FtpExceptionCanNotHaveDataConnection(e.getMessage());
    // }
    // }

    socket = _socketFactory_.createSocket(__passiveHost, __passivePort);

    if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) {
      socket.close();
      return null;
    }

    if (__remoteVerificationEnabled && !verifyRemote(socket)) {
      InetAddress host1, host2;

      host1 = socket.getInetAddress();
      host2 = getRemoteAddress();

      socket.close();

      // our precaution
      throw new FtpExceptionCanNotHaveDataConnection(
          "Host attempting data connection " + host1.getHostAddress()
              + " is not same as server " + host2.getHostAddress()
              + " So we intentionally close it for security precaution.");
    }

    if (__dataTimeout >= 0)
      socket.setSoTimeout(__dataTimeout);

    return socket;
  }

  /***
   * Sets the timeout in milliseconds to use for data connection. set
   * immediately after opening the data connection.
   ***/
  public void setDataTimeout(int timeout) {
    __dataTimeout = timeout;
  }

  /***
   * Closes the connection to the FTP server and restores connection parameters
   * to the default values.
   * <p>
   * 
   * @exception IOException
   *              If an error occurs while disconnecting.
   ***/
  public void disconnect() throws IOException {
    __initDefaults();
    super.disconnect();
    // no worry for data connection, since we always close it
    // in every ftp command that invloves data connection
  }

  /***
   * Enable or disable verification that the remote host taking part of a data
   * connection is the same as the host to which the control connection is
   * attached. The default is for verification to be enabled. You may set this
   * value at any time, whether the FTPClient is currently connected or not.
   * <p>
   * 
   * @param enable
   *          True to enable verification, false to disable verification.
   ***/
  public void setRemoteVerificationEnabled(boolean enable) {
    __remoteVerificationEnabled = enable;
  }

  /***
   * Return whether or not verification of the remote host participating in data
   * connections is enabled. The default behavior is for verification to be
   * enabled.
   * <p>
   * 
   * @return True if verification is enabled, false if not.
   ***/
  public boolean isRemoteVerificationEnabled() {
    return __remoteVerificationEnabled;
  }

  /***
   * Login to the FTP server using the provided username and password.
   * <p>
   * 
   * @param username
   *          The username to login under.
   * @param password
   *          The password to use.
   * @return True if successfully completed, false if not.
   * @exception FTPConnectionClosedException
   *              If the FTP server prematurely closes the connection as a
   *              result of the client being idle or some other reason causing
   *              the server to send FTP reply code 421. This exception may be
   *              caught either as an IOException or independently as itself.
   * @exception IOException
   *              If an I/O error occurs while either sending a command to the
   *              server or receiving a reply from the server.
   ***/
  public boolean login(String username, String password) throws IOException {
    user(username);

    if (FTPReply.isPositiveCompletion(getReplyCode()))
      return true;

    // If we get here, we either have an error code, or an intermmediate
    // reply requesting password.
    if (!FTPReply.isPositiveIntermediate(getReplyCode()))
      return false;

    return FTPReply.isPositiveCompletion(pass(password));
  }

  /***
   * Logout of the FTP server by sending the QUIT command.
   * <p>
   * 
   * @return True if successfully completed, false if not.
   * @exception FTPConnectionClosedException
   *              If the FTP server prematurely closes the connection as a
   *              result of the client being idle or some other reason causing
   *              the server to send FTP reply code 421. This exception may be
   *              caught either as an IOException or independently as itself.
   * @exception IOException
   *              If an I/O error occurs while either sending a command to the
   *              server or receiving a reply from the server.
   ***/
  public boolean logout() throws IOException {
    return FTPReply.isPositiveCompletion(quit());
  }

  /**
   * Retrieve a list reply for path
   * 
   * @param path
   * @param entries
   * @param limit
   * @param parser
   * @throws IOException
   * @throws FtpExceptionCanNotHaveDataConnection
   * @throws FtpExceptionUnknownForcedDataClose
   * @throws FtpExceptionControlClosedByForcedDataClose
   */
  public void retrieveList(String path, List<FTPFile> entries, int limit,
      FTPFileEntryParser parser) throws IOException,
      FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose,
      FtpExceptionControlClosedByForcedDataClose {
    Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path);

    if (socket == null)
      throw new FtpExceptionCanNotHaveDataConnection("LIST "
          + ((path == null) ? "" : path));

    BufferedReader reader = new BufferedReader(new InputStreamReader(
        socket.getInputStream()));

    // force-close data channel socket, when download limit is reached
    // boolean mandatory_close = false;

    // List entries = new LinkedList();
    int count = 0;
    String line = parser.readNextEntry(reader);
    while (line != null) {
      FTPFile ftpFile = parser.parseFTPEntry(line);
      // skip non-formatted lines
      if (ftpFile == null) {
        line = parser.readNextEntry(reader);
        continue;
      }
      entries.add(ftpFile);
      count += line.length();
      // impose download limit if limit >= 0, otherwise no limit
      // here, cut off is up to the line when total bytes is just over limit
      if (limit >= 0 && count > limit) {
        // mandatory_close = true;
        break;
      }
      line = parser.readNextEntry(reader);
    }

    // if (mandatory_close)
    // you always close here, no matter mandatory_close or not.
    // however different ftp servers respond differently, see below.
    socket.close();

    // scenarios:
    // (1) mandatory_close is false, download limit not reached
    // no special care here
    // (2) mandatory_close is true, download limit is reached
    // different servers have different reply codes:

    try {
      int reply = getReply();
      if (!_notBadReply(reply))
        throw new FtpExceptionUnknownForcedDataClose(getReplyString());
    } catch (FTPConnectionClosedException e) {
      // some ftp servers will close control channel if data channel socket
      // is closed by our end before all data has been read out. Check:
      // tux414.q-tam.hp.com FTP server (hp.com version whp02)
      // so must catch FTPConnectionClosedException thrown by getReply() above
      // disconnect();
      throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
    }

  }

  /**
   * Retrieve a file for path
   * 
   * @param path
   * @param os
   * @param limit
   * @throws IOException
   * @throws FtpExceptionCanNotHaveDataConnection
   * @throws FtpExceptionUnknownForcedDataClose
   * @throws FtpExceptionControlClosedByForcedDataClose
   */
  public void retrieveFile(String path, OutputStream os, int limit)
      throws IOException, FtpExceptionCanNotHaveDataConnection,
      FtpExceptionUnknownForcedDataClose,
      FtpExceptionControlClosedByForcedDataClose {

    Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path);

    if (socket == null)
      throw new FtpExceptionCanNotHaveDataConnection("RETR "
          + ((path == null) ? "" : path));

    InputStream input = socket.getInputStream();

    // 20040318, xing, treat everything as BINARY_FILE_TYPE for now
    // do we ever need ASCII_FILE_TYPE?
    // if (__fileType == ASCII_FILE_TYPE)
    // input = new FromNetASCIIInputStream(input);

    // fixme, should we instruct server here for binary file type?

    // force-close data channel socket
    // boolean mandatory_close = false;

    int len;
    int count = 0;
    byte[] buf = new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE];
    while ((len = input.read(buf, 0, buf.length)) != -1) {
      count += len;
      // impose download limit if limit >= 0, otherwise no limit
      // here, cut off is exactly of limit bytes
      if (limit >= 0 && count > limit) {
        os.write(buf, 0, len - (count - limit));
        // mandatory_close = true;
        break;
      }
      os.write(buf, 0, len);
      os.flush();
    }

    // if (mandatory_close)
    // you always close here, no matter mandatory_close or not.
    // however different ftp servers respond differently, see below.
    socket.close();

    // scenarios:
    // (1) mandatory_close is false, download limit not reached
    // no special care here
    // (2) mandatory_close is true, download limit is reached
    // different servers have different reply codes:

    // do not need this
    // sendCommand("ABOR");

    try {
      int reply = getReply();
      if (!_notBadReply(reply))
        throw new FtpExceptionUnknownForcedDataClose(getReplyString());
    } catch (FTPConnectionClosedException e) {
      // some ftp servers will close control channel if data channel socket
      // is closed by our end before all data has been read out. Check:
      // tux414.q-tam.hp.com FTP server (hp.com version whp02)
      // so must catch FTPConnectionClosedException thrown by getReply() above
      // disconnect();
      throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage());
    }

  }

  // reply check after closing data connection
  private boolean _notBadReply(int reply) {

    if (FTPReply.isPositiveCompletion(reply)) {
      // do nothing
    } else if (reply == 426) { // FTPReply.TRANSFER_ABORTED
      // some ftp servers reply 426, e.g.,
      // foggy FTP server (Version wu-2.6.2(2)
      // there is second reply witing? no!
      // getReply();
    } else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN
      // some ftp servers reply 450, e.g.,
      // ProFTPD [ftp.kernel.org]
      // there is second reply witing? no!
      // getReply();
    } else if (reply == 451) { // FTPReply.ACTION_ABORTED
      // some ftp servers reply 451, e.g.,
      // ProFTPD [ftp.kernel.org]
      // there is second reply witing? no!
      // getReply();
    } else if (reply == 451) { // FTPReply.ACTION_ABORTED
    } else {
      // what other kind of ftp server out there?
      return false;
    }

    return true;
  }

  /***
   * Sets the file type to be transferred. This should be one of
   * <code> FTP.ASCII_FILE_TYPE </code>, <code> FTP.IMAGE_FILE_TYPE </code>,
   * etc. The file type only needs to be set when you want to change the type.
   * After changing it, the new type stays in effect until you change it again.
   * The default file type is <code> FTP.ASCII_FILE_TYPE </code> if this method
   * is never called.
   * <p>
   * 
   * @param fileType
   *          The <code> _FILE_TYPE </code> constant indcating the type of file.
   * @return True if successfully completed, false if not.
   * @exception FTPConnectionClosedException
   *              If the FTP server prematurely closes the connection as a
   *              result of the client being idle or some other reason causing
   *              the server to send FTP reply code 421. This exception may be
   *              caught either as an IOException or independently as itself.
   * @exception IOException
   *              If an I/O error occurs while either sending a command to the
   *              server or receiving a reply from the server.
   ***/
  public boolean setFileType(int fileType) throws IOException {
    if (FTPReply.isPositiveCompletion(type(fileType))) {
      /*
       * __fileType = fileType; __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
       */
      return true;
    }
    return false;
  }

  /***
   * Fetches the system type name from the server and returns the string. This
   * value is cached for the duration of the connection after the first call to
   * this method. In other words, only the first time that you invoke this
   * method will it issue a SYST command to the FTP server. FTPClient will
   * remember the value and return the cached value until a call to disconnect.
   * <p>
   * 
   * @return The system type name obtained from the server. null if the
   *         information could not be obtained.
   * @exception FTPConnectionClosedException
   *              If the FTP server prematurely closes the connection as a
   *              result of the client being idle or some other reason causing
   *              the server to send FTP reply code 421. This exception may be
   *              caught either as an IOException or independently as itself.
   * @exception IOException
   *              If an I/O error occurs while either sending a command to the
   *              server or receiving a reply from the server.
   ***/
  public String getSystemName() throws IOException, FtpExceptionBadSystResponse {
    // if (syst() == FTPReply.NAME_SYSTEM_TYPE)
    // Technically, we should expect a NAME_SYSTEM_TYPE response, but
    // in practice FTP servers deviate, so we soften the condition to
    // a positive completion.
    if (__systemName == null && FTPReply.isPositiveCompletion(syst())) {
      __systemName = (getReplyStrings()[0]).substring(4);
    } else {
      throw new FtpExceptionBadSystResponse("Bad response of SYST: "
          + getReplyString());
    }

    return __systemName;
  }

  /***
   * Sends a NOOP command to the FTP server. This is useful for preventing
   * server timeouts.
   * <p>
   * 
   * @return True if successfully completed, false if not.
   * @exception FTPConnectionClosedException
   *              If the FTP server prematurely closes the connection as a
   *              result of the client being idle or some other reason causing
   *              the server to send FTP reply code 421. This exception may be
   *              caught either as an IOException or independently as itself.
   * @exception IOException
   *              If an I/O error occurs while either sending a command to the
   *              server or receiving a reply from the server.
   ***/
  public boolean sendNoOp() throws IOException {
    return FTPReply.isPositiveCompletion(noop());
  }

  // client.stat(path);
  // client.sendCommand("STAT");
  // client.sendCommand("STAT",path);
  // client.sendCommand("MDTM",path);
  // client.sendCommand("SIZE",path);
  // client.sendCommand("HELP","SITE");
  // client.sendCommand("SYST");
  // client.setRestartOffset(120);

}
