kaldi-io-test.cc

Go to the documentation of this file.

// util/kaldi-io-test.cc

// Copyright 2009-2011  Microsoft Corporation

// See ../../COPYING for clarification regarding multiple authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//  http://www.apache.org/licenses/LICENSE-2.0

// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABLITY OR NON-INFRINGEMENT.
// See the Apache 2 License for the specific language governing permissions and
// limitations under the License.
#ifndef _MSC_VER
#include <unistd.h>
#endif
#include "base/io-funcs.h"
#include "util/kaldi-io.h"
#include "base/kaldi-math.h"
#include "base/kaldi-utils.h"

namespace kaldi {



void UnitTestClassifyRxfilename() {
  KALDI_ASSERT(ClassifyRxfilename("") == kStandardInput);
  KALDI_ASSERT(ClassifyRxfilename(" ") == kNoInput);
  KALDI_ASSERT(ClassifyRxfilename(" a ") == kNoInput);
  KALDI_ASSERT(ClassifyRxfilename("a ") == kNoInput);
  KALDI_ASSERT(ClassifyRxfilename("a") == kFileInput);
  KALDI_ASSERT(ClassifyRxfilename("-") == kStandardInput);
  KALDI_ASSERT(ClassifyRxfilename("b|") == kPipeInput);
  KALDI_ASSERT(ClassifyRxfilename("|b") == kNoInput);
  KALDI_ASSERT(ClassifyRxfilename("b c|") == kPipeInput);
  KALDI_ASSERT(ClassifyRxfilename(" b c|") == kPipeInput);
  KALDI_ASSERT(ClassifyRxfilename("a b c:123") == kOffsetFileInput);
  KALDI_ASSERT(ClassifyRxfilename("a b c:3") == kOffsetFileInput);
  KALDI_ASSERT(ClassifyRxfilename("a b c:") == kFileInput);
  KALDI_ASSERT(ClassifyRxfilename("a b c/3") == kFileInput);
  KALDI_ASSERT(ClassifyRxfilename("ark,s,cs:a b c") == kNoInput);
  KALDI_ASSERT(ClassifyRxfilename("scp:a b c") == kNoInput);

}


void UnitTestClassifyWxfilename() {
  KALDI_ASSERT(ClassifyWxfilename("") == kStandardOutput);
  KALDI_ASSERT(ClassifyWxfilename(" ") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename(" a ") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("a ") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("a") == kFileOutput);
  KALDI_ASSERT(ClassifyWxfilename("-") == kStandardOutput);
  KALDI_ASSERT(ClassifyWxfilename("b|") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("|b") == kPipeOutput);
  KALDI_ASSERT(ClassifyWxfilename("| b ") == kPipeOutput);
  KALDI_ASSERT(ClassifyWxfilename("b c|") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("a b c:123") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("ark,s,cs:a b c") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("scp:a b c") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("a b c:3") == kNoOutput);
  KALDI_ASSERT(ClassifyWxfilename("a b c:") == kFileOutput);
  KALDI_ASSERT(ClassifyWxfilename("a b c/3") == kFileOutput);
}

void UnitTestIoNew(bool binary) {
  {
    const char *filename = "tmpf";

    Output ko(filename, binary);
    std::ostream &outfile = ko.Stream();
    if (!binary) outfile << "\t";
    int64 i1 = Rand() % 10000;
    WriteBasicType(outfile, binary, i1);
    uint16 i2 = Rand() % 10000;
    WriteBasicType(outfile, binary, i2);
    if (!binary) outfile << "\t";
    char c = Rand();
    WriteBasicType(outfile, binary, c);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::vector<int32> vec1;
    WriteIntegerVector(outfile, binary, vec1);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::vector<uint16> vec2;
    for (size_t i = 0; i < 10; i++) vec2.push_back(Rand()%100 - 10);
    WriteIntegerVector(outfile, binary, vec2);
    if (!binary) outfile << " \n";
    std::vector<char> vec3;
    for (size_t i = 0; i < 10; i++) vec3.push_back(Rand()%100);
    WriteIntegerVector(outfile, binary, vec3);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    const char *token1 = "Hi";
    WriteToken(outfile, binary, token1);
    if (!binary) outfile << " \n";
    std::string token2 = "There.";
    WriteToken(outfile, binary, token2);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::string token3 = "You.";
    WriteToken(outfile, binary, token3);
    if (!binary && Rand()%2 == 0) outfile << " ";
    float f1 = RandUniform();
    WriteBasicType(outfile, binary, f1);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    float f2 = RandUniform();
    WriteBasicType(outfile, binary, f2);
    double d1 = RandUniform();
    WriteBasicType(outfile, binary, d1);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    double d2 = RandUniform();
    WriteBasicType(outfile, binary, d2);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    ko.Close();

    {
      bool binary_in;
      Input ki(filename, &binary_in);
      std::istream &infile = ki.Stream();
      int64 i1_in;
      ReadBasicType(infile, binary_in, &i1_in);
      KALDI_ASSERT(i1_in == i1);
      uint16 i2_in;
      ReadBasicType(infile, binary_in, &i2_in);
      KALDI_ASSERT(i2_in == i2);
      char c_in;
      ReadBasicType(infile, binary_in, &c_in);
      KALDI_ASSERT(c_in == c);
      std::vector<int32> vec1_in;
      ReadIntegerVector(infile, binary_in, &vec1_in);
      KALDI_ASSERT(vec1_in == vec1);
      std::vector<uint16> vec2_in;
      ReadIntegerVector(infile, binary_in, &vec2_in);
      KALDI_ASSERT(vec2_in == vec2);
      std::vector<char> vec3_in;
      ReadIntegerVector(infile, binary_in, &vec3_in);
      KALDI_ASSERT(vec3_in == vec3);
      std::string  token1_in, token2_in;
      KALDI_ASSERT(Peek(infile, binary_in) == static_cast<int>(*token1));
      ReadToken(infile, binary_in, &token1_in);
      KALDI_ASSERT(token1_in == (std::string)token1);
      ReadToken(infile, binary_in, &token2_in);
      KALDI_ASSERT(token2_in == token2);
      if (Rand() % 2 == 0)
        ExpectToken(infile, binary_in, token3.c_str());
      else
        ExpectToken(infile, binary_in, token3);
      float f1_in;  // same type.
      ReadBasicType(infile, binary_in, &f1_in);
      AssertEqual(f1_in, f1);
      double f2_in;  // wrong type.
      ReadBasicType(infile, binary_in, &f2_in);
      AssertEqual(f2_in, f2);
      double d1_in;  // same type.
      ReadBasicType(infile, binary_in, &d1_in);
      AssertEqual(d1_in, d1);
      float d2_in;  // wrong type.
      ReadBasicType(infile, binary_in, &d2_in);
      AssertEqual(d2_in, d2);
      KALDI_ASSERT(Peek(infile, binary_in) == -1);
    }
    unlink(filename);
  }
}

void UnitTestIoPipe(bool binary) {
  // This is as UnitTestIoNew except with different filenames.
  {
#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
    // self-invocation on Windows that emulates cat(1)
    const char *filename_out = "|kaldi-io-test cat > tmpf.gz",
        *filename_in = "kaldi-io-test cat tmpf.gz|";
#else
    const char *filename_out = "|gzip -c > tmpf.gz",
        *filename_in = "gunzip -c tmpf.gz |";
#endif

    Output ko(filename_out, binary);
    std::ostream &outfile = ko.Stream();
    if (!binary) outfile << "\t";
    int64 i1 = Rand() % 10000;
    WriteBasicType(outfile, binary, i1);
    uint16 i2 = Rand() % 10000;
    WriteBasicType(outfile, binary, i2);
    if (!binary) outfile << "\t";
    char c = Rand();
    WriteBasicType(outfile, binary, c);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::vector<int32> vec1;
    WriteIntegerVector(outfile, binary, vec1);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::vector<uint16> vec2;
    for (size_t i = 0; i < 10; i++) vec2.push_back(Rand()%100 - 10);
    WriteIntegerVector(outfile, binary, vec2);
    if (!binary) outfile << " \n";
    WriteToken(outfile, binary, "<foo>");
    std::vector<char> vec3;
    for (size_t i = 0; i < 10; i++) vec3.push_back(Rand()%100);
    WriteIntegerVector(outfile, binary, vec3);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    const char *token1 = "Hi";
    WriteToken(outfile, binary, token1);
    if (!binary) outfile << " \n";
    std::string token2 = "There.";
    WriteToken(outfile, binary, token2);
    if (!binary && Rand()%2 == 0) outfile << " \n";
    std::string token3 = "You.";
    WriteToken(outfile, binary, token3);
    if (!binary && Rand()%2 == 0) outfile << " ";
    float f1 = RandUniform();
    WriteBasicType(outfile, binary, f1);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    float f2 = RandUniform();
    WriteBasicType(outfile, binary, f2);
    double d1 = RandUniform();
    WriteBasicType(outfile, binary, d1);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    double d2 = RandUniform();
    WriteBasicType(outfile, binary, d2);
    if (!binary && Rand()%2 == 0) outfile << "\t";
    bool ans = ko.Close();
    KALDI_ASSERT(ans);
#ifndef _MSC_VER
    Sleep(1);  // This test does not work without this sleep:
    // seems to be some kind of file-system latency.
#endif
    {
      bool binary_in;
      Input ki(filename_in, &binary_in);
      std::istream &infile = ki.Stream();
      int64 i1_in;
      ReadBasicType(infile, binary_in, &i1_in);
      KALDI_ASSERT(i1_in == i1);
      uint16 i2_in;
      ReadBasicType(infile, binary_in, &i2_in);
      KALDI_ASSERT(i2_in == i2);
      char c_in;
      ReadBasicType(infile, binary_in, &c_in);
      KALDI_ASSERT(c_in == c);
      std::vector<int32> vec1_in;
      ReadIntegerVector(infile, binary_in, &vec1_in);
      KALDI_ASSERT(vec1_in == vec1);
      std::vector<uint16> vec2_in;
      ReadIntegerVector(infile, binary_in, &vec2_in);
      KALDI_ASSERT(vec2_in == vec2);
      std::vector<char> vec3_in;
      KALDI_ASSERT(PeekToken(infile, binary_in) == static_cast<int>('f'));
      ExpectToken(infile, binary_in, "<foo>");
      ReadIntegerVector(infile, binary_in, &vec3_in);
      KALDI_ASSERT(vec3_in == vec3);
      std::string  token1_in, token2_in;
      KALDI_ASSERT(Peek(infile, binary_in) == static_cast<int>(*token1));
      ReadToken(infile, binary_in, &token1_in);
      KALDI_ASSERT(token1_in == (std::string)token1);
      ReadToken(infile, binary_in, &token2_in);
      KALDI_ASSERT(token2_in == token2);
      if (Rand() % 2 == 0)
        ExpectToken(infile, binary_in, token3.c_str());
      else
        ExpectToken(infile, binary_in, token3);
      float f1_in;  // same type.
      ReadBasicType(infile, binary_in, &f1_in);
      AssertEqual(f1_in, f1);
      double f2_in;  // wrong type.
      ReadBasicType(infile, binary_in, &f2_in);
      AssertEqual(f2_in, f2);
      double d1_in;  // same type.
      ReadBasicType(infile, binary_in, &d1_in);
      AssertEqual(d1_in, d1);
      float d2_in;  // wrong type.
      ReadBasicType(infile, binary_in, &d2_in);
      AssertEqual(d2_in, d2);
      KALDI_ASSERT(Peek(infile, binary_in) == -1);
    }
  }
  unlink("tmpf.txt");
  unlink("tmpf.gz");
}

void UnitTestIoStandard() {
  /*
    Don't do the the following part because it requires
    to pipe from an empty file, for it to not hang.
  {
    Input inp("", NULL);  // standard input.
    KALDI_ASSERT(inp.Stream().get() == -1);
  }
  {
    Input inp("-", NULL);  // standard input.
    KALDI_ASSERT(inp.Stream().get() == -1);
    }*/

  {
    std::cout << "Should see: foo\n";
    Output out("", false);
    out.Stream() << "foo\n";
  }
  {
    std::cout << "Should see: bar\n";
    Output out("-", false);
    out.Stream() << "bar\n";
  }
}

// This is Windows-specific.
void UnitTestNativeFilename() {
#ifdef KALDI_CYGWIN_COMPAT
  extern std::string MapCygwinPath(const std::string &filename);

  KALDI_ASSERT(MapCygwinPath("") == "");
  KALDI_ASSERT(MapCygwinPath(".") == ".");
  KALDI_ASSERT(MapCygwinPath("..") == "..");
  KALDI_ASSERT(MapCygwinPath("/dev/null")[0] != '/');
  KALDI_ASSERT(MapCygwinPath("/tmp")[1] == ':');
  KALDI_ASSERT(MapCygwinPath("/tmp/")[1] == ':');
  KALDI_ASSERT(MapCygwinPath("/tmp/foo")[1] == ':');
  KALDI_ASSERT(MapCygwinPath("/cygdrive/c") == "c:/");
  KALDI_ASSERT(MapCygwinPath("/cygdrive/c/") == "c:/");
  KALDI_ASSERT(MapCygwinPath("/cygdrive/c/foo") == "c:/foo");
#endif
}

}  // end namespace kaldi.

#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
// Windows has no cat!  There is probably no suitable tool to test popen I/O on
// Windows, so we emulate a lame version of cat(1).
static int TinyCat(int argc, const char** argv) {
  const char* name_in = argc > 0 && strcmp(argv[0], "-") ? argv[0] : NULL;
  int fd_in = name_in ? _open(name_in, _O_RDONLY) : _fileno(stdin);
  if (fd_in < 0)
    return 1;

  int fd_out = _fileno(stdout);
  _setmode(fd_in, _O_BINARY);
  _setmode(fd_out, _O_BINARY);

  char buffer[100];
  int last_read;
  while ((last_read = _read(fd_in, buffer, sizeof(buffer))) > 0)
    _write(fd_out, buffer, last_read);

  if (name_in) _close(fd_in);
  return 0;
}
#endif

int main(int argc, const char** argv) {
  using namespace kaldi;
#if defined(_MSC_VER) && !defined(KALDI_CYGWIN_COMPAT)
  if (argc > 1 && strcmp(argv[1], "cat") == 0)
    return TinyCat(argc - 2, argv + 2);
#endif

  UnitTestNativeFilename();
  UnitTestIoNew(false);
  UnitTestIoNew(true);
  UnitTestIoPipe(true);
  UnitTestIoPipe(false);
  UnitTestIoStandard();
  UnitTestClassifyRxfilename();
  UnitTestClassifyWxfilename();

  KALDI_ASSERT(1);  // just wanted to check that KALDI_ASSERT does not fail
  // for 1.
  return 0;
}