/***********************************************************************************************************************
 *
 * Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 **********************************************************************************************************************/

package eu.stratosphere.nephele.services.iomanager;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Random;

import junit.framework.Assert;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import eu.stratosphere.nephele.io.IOReadableWritable;
import eu.stratosphere.nephele.services.iomanager.Channel;
import eu.stratosphere.nephele.services.iomanager.IOManager;
import eu.stratosphere.nephele.services.memorymanager.DefaultMemoryManagerTest;
import eu.stratosphere.nephele.services.memorymanager.MemorySegment;
import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager;
import eu.stratosphere.nephele.template.AbstractInvokable;


/**
 * Integration test case for the I/O manager.
 *
 *
 * @author Stephan Ewen (stephan.ewen@tu-berlin.de)
 */
public class IOManagerITCase {
	
	private static final Log LOG = LogFactory.getLog(IOManagerITCase.class);
	
	private static final long SEED = 649180756312423613L;

	private static final int NUMBER_OF_SEGMENTS = 10; // 10

	private static final int SEGMENT_SIZE = 1024 * 1024; // 1M
	
	private final int NUM_CHANNELS = 29;
	
	private final int NUMBERS_TO_BE_WRITTEN = NUM_CHANNELS * 1000000;

	private IOManager ioManager;

	private DefaultMemoryManager memoryManager;


	@Before
	public void beforeTest() {
		memoryManager = new DefaultMemoryManager(NUMBER_OF_SEGMENTS * SEGMENT_SIZE);
		ioManager = new IOManager();
	}

	@After
	public void afterTest() throws Exception {
		ioManager.shutdown();
		Assert.assertTrue("IO Manager has not properly shut down.", ioManager.isProperlyShutDown());
		
		Assert.assertTrue("Not all memory was returned to the memory manager in the test.", memoryManager.verifyEmpty());
		memoryManager.shutdown();
		memoryManager = null;
	}

	// ------------------------------------------------------------------------
	
	/**
	 * This test instantiates multiple channels and writes to them in parallel and re-reads the data in 
	 * parallel. It is designed to check the ability of the IO manager to correctly handle multiple threads.
	 */
	@Test
	public void parallelChannelsTest() throws Exception
	{
		LOG.info("Starting parallel channels test.");
		
		final Random rnd = new Random(SEED);
		final AbstractInvokable memOwner = new DefaultMemoryManagerTest.DummyInvokable();
		
		Channel.ID[] ids = new Channel.ID[NUM_CHANNELS];
		BlockChannelWriter[] writers = new BlockChannelWriter[NUM_CHANNELS];
		BlockChannelReader[] readers = new BlockChannelReader[NUM_CHANNELS];
		ChannelWriterOutputView[] outs = new ChannelWriterOutputView[NUM_CHANNELS];
		ChannelReaderInputView[] ins = new ChannelReaderInputView[NUM_CHANNELS];
		
		int[] writingCounters = new int[NUM_CHANNELS];
		int[] readingCounters = new int[NUM_CHANNELS];
		
		// instantiate the channels and writers
		for (int i = 0; i < NUM_CHANNELS; i++)
		{
			ids[i] = this.ioManager.createChannel();
			writers[i] = this.ioManager.createBlockChannelWriter(ids[i]);
			
			List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2);
			outs[i] = new ChannelWriterOutputView(writers[i], memSegs, this.memoryManager.getPageSize());
		}
		
		
		Value val = new Value();
		
		// write a lot of values unevenly distributed over the channels
		int nextLogCount = 0;
		float nextLogFraction = 0.0f;
		
		LOG.info("Writing to channels...");
		for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
			
			if (i == nextLogCount) {
				LOG.info("... " + (int) (nextLogFraction * 100) + "% done.");
				nextLogFraction += 0.05;
				nextLogCount = (int) (nextLogFraction * NUMBERS_TO_BE_WRITTEN);
			}
			
			int channel = skewedSample(rnd, NUM_CHANNELS - 1);
			
			val.value = String.valueOf(writingCounters[channel]++);
			val.write(outs[channel]);
		}
		LOG.info("Writing done, flushing contents...");
		
		// close all writers
		for (int i = 0; i < NUM_CHANNELS; i++) {
			this.memoryManager.release(outs[i].close());
		}
		outs = null;
		writers = null;
		
		// instantiate the readers for sequential read
		LOG.info("Reading channels sequentially...");
		for (int i = 0; i < NUM_CHANNELS; i++)
		{
			List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2);
			
			LOG.info("Reading channel " + (i+1) + "/" + NUM_CHANNELS + '.');
				
			final BlockChannelReader reader = this.ioManager.createBlockChannelReader(ids[i]);
			final ChannelReaderInputView in = new ChannelReaderInputView(reader, memSegs, false);
			int nextVal = 0;
			
			try {
				while (true) {
					val.read(in);
					int intValue = 0;
					try {
						intValue = Integer.parseInt(val.value);
					}
					catch (NumberFormatException nfex) {
						Assert.fail("Invalid value read from reader. Valid decimal number expected.");
					}
					Assert.assertEquals("Written and read values do not match during sequential read.", nextVal, intValue);
					nextVal++;
				}
			} catch (EOFException eofex) {
				// expected
			}
			
			Assert.assertEquals("NUmber of written numbers differs from number of read numbers.", writingCounters[i], nextVal);
			
			this.memoryManager.release(in.close());
		}
		LOG.info("Sequential reading done.");
		
		// instantiate the readers
		LOG.info("Reading channels randomly...");
		for (int i = 0; i < NUM_CHANNELS; i++) {
			
			List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2);
				
			readers[i] = this.ioManager.createBlockChannelReader(ids[i]);
			ins[i] = new ChannelReaderInputView(readers[i], memSegs, false);
		}
		
		nextLogCount = 0;
		nextLogFraction = 0.0f;
		
		// read a lot of values in a mixed order from the channels
		for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
			
			if (i == nextLogCount) {
				LOG.info("... " + (int) (nextLogFraction * 100) + "% done.");
				nextLogFraction += 0.05;
				nextLogCount = (int) (nextLogFraction * NUMBERS_TO_BE_WRITTEN);
			}
			
			while (true) {
				final int channel = skewedSample(rnd, NUM_CHANNELS - 1);
				if (ins[channel] != null) {
					try {
						val.read(ins[channel]);
						int intValue;
						try {
							intValue = Integer.parseInt(val.value);
						}
						catch (NumberFormatException nfex) {
							Assert.fail("Invalid value read from reader. Valid decimal number expected.");
							return;
						}
						
						Assert.assertEquals("Written and read values do not match.", readingCounters[channel]++, intValue);
						
						break;
					} catch (EOFException eofex) {
						this.memoryManager.release(ins[channel].close());
						ins[channel] = null;
					}
				}
			}
			
		}
		LOG.info("Random reading done.");
		
		// close all readers
		for (int i = 0; i < NUM_CHANNELS; i++) {
			if (ins[i] != null) {
				this.memoryManager.release(ins[i].close());
			}
			readers[i].closeAndDelete();
		}
		
		ins = null;
		readers = null;
		
		// check that files are deleted
		for (int i = 0; i < NUM_CHANNELS; i++) {
			File f = new File(ids[i].getPath());
			Assert.assertFalse("Channel file has not been deleted.", f.exists());
		}
	}
	
	private static final int skewedSample(Random rnd, int max) {
		double uniform = rnd.nextDouble();
		double var = Math.pow(uniform, 8.0);
		double pareto = 0.2 / var;
		
		int val = (int) pareto;
		return val > max ? val % max : val;
	}
	
	
	// ------------------------------------------------------------------------
	
	protected static class Value implements IOReadableWritable {

		String value;

		public Value() {
		}

		public Value(String val) {
			this.value = val;
		}

		@Override
		public void read(DataInput in) throws IOException {
			value = in.readUTF();
		}

		@Override
		public void write(DataOutput out) throws IOException {
			out.writeUTF(this.value);
		}

		@Override
		public int hashCode() {
			final int prime = 31;
			int result = 1;
			result = prime * result + ((value == null) ? 0 : value.hashCode());
			return result;
		}

		@Override
		public boolean equals(Object obj) {
			if (this == obj)
				return true;
			if (obj == null)
				return false;
			if (getClass() != obj.getClass())
				return false;
			Value other = (Value) obj;

			if (value == null) {
				if (other.value != null)
					return false;
			} else if (!value.equals(other.value))
				return false;
			return true;
		}
	}

}
