From fd85dc1c1326d5baba95228f6088f74a07d61dc3 Mon Sep 17 00:00:00 2001 From: Ksenia Proskurina Date: Sat, 17 Dec 2016 21:58:03 +0300 Subject: [PATCH 1/3] updatind --- .../g596/proskurina/task2/FileWorker.java | 163 +++++++++-- .../task2/ImplementationKeyValueStorage.java | 268 +++++++++++++----- .../ImplementationKeyValueStorageTest.java | 9 +- 3 files changed, 345 insertions(+), 95 deletions(-) diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java index f8cd6066c..c9defadf8 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java @@ -1,50 +1,165 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; import java.io.*; +import java.nio.ByteBuffer; /** * Created by Lenovo on 31.10.2016. */ -public class FileWorker { +public class FileWorker implements Closeable{ - private static void exists(String fileName) throws FileNotFoundException { - File file = new File(fileName); - if (!file.exists()) { - throw new FileNotFoundException(file.getName()); + private final File file; + private final String fileName; + + private InputStream readBuffer = null; + private OutputStream writeBuffer = null; + + private long currentPositionInStream = 0; + + public FileWorker(String fileName) { + this.file = new File(fileName); + this.fileName = fileName; + } + + public void createFile() { + try { + file.createNewFile(); + } catch (IOException e) { + System.out.println("file didn't created"); + throw new RuntimeException(e); } } - public static String read(String fileName) throws FileNotFoundException { + public void rename(String newName) { + File newFile = new File(newName); + file.renameTo(newFile); + } - StringBuffer inputData = new StringBuffer(); - exists(fileName); - File file = new File(fileName); + public void appendMode() { + try { + close(); + writeBuffer = new FileOutputStream(file.getAbsoluteFile(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public boolean exist() { + return file.exists(); + } + + private boolean innerExist() throws FileNotFoundException { + if (!exist()) { + throw new FileNotFoundException(fileName); + } + return true; + } - try (BufferedReader in = new BufferedReader(new FileReader(file.getAbsoluteFile()))) { - String s; - while ((s = in.readLine()) != null) { - inputData.append(s); - inputData.append("\n"); + public void writeSubmit() { + if (writeBuffer != null) { + try { + writeBuffer.flush(); + writeBuffer.close(); + writeBuffer = null; + } catch (IOException e) { + throw new RuntimeException(e); } + } + } + public long write(String str) { + try { + innerExist(); + if (writeBuffer == null) { + writeBuffer = new BufferedOutputStream(new FileOutputStream(file.getAbsoluteFile())); + } + byte[] bytes = str.getBytes(); + byte[] bytesNumber = ByteBuffer.allocate(4).putInt(bytes.length).array(); + writeBuffer.write(bytesNumber); + writeBuffer.write(bytes); + return 4 + bytes.length; } catch (IOException e) { throw new RuntimeException(e); } - return inputData.toString(); } - public static void write(String fileName, String text) { + public String read() { + try { + innerExist(); + if (readBuffer == null) { + readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); + currentPositionInStream = 0; + } + if (readBuffer.available() < 4) { + readBuffer.close(); + readBuffer = null; + return null; + } + byte[] bytesNumberArray = new byte[4]; + int bytesNumberArraySize = readBuffer.read(bytesNumberArray, 0, 4); + //addToCalc(bytes); + if (bytesNumberArraySize < 4) { + readBuffer.close(); + throw new RuntimeException("Error in reading bytes number"); + } + currentPositionInStream += bytesNumberArraySize; + int bytesNumber = ByteBuffer.wrap(bytesNumberArray).getInt(); + byte[] bytesArray = new byte[bytesNumber]; + int bytesArraySize = readBuffer.read(bytesArray, 0, bytesNumber); + //addToCalc(bytes); + if (bytesArraySize < bytesNumber) { + readBuffer.close(); + throw new RuntimeException("Error in reading bytes"); + } + currentPositionInStream += bytesArraySize; + return new String(bytesArray); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public long fileLength() { + try (FileInputStream inputStream = new FileInputStream(file.getAbsoluteFile())) { + return inputStream.available(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - File file = new File(fileName); + public void goToPosition(long position) { try { - if (!file.exists()) { - file.createNewFile(); + innerExist(); + if (readBuffer == null || currentPositionInStream > position) { + if (readBuffer != null) { + readBuffer.close(); + } + readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); + currentPositionInStream = 0; } - PrintWriter out = new PrintWriter(file.getAbsoluteFile()); - try { - out.print(text); - } finally { - out.close(); + while (currentPositionInStream < position) { + currentPositionInStream += readBuffer.skip(position - currentPositionInStream); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void delete() { + file.delete(); + } + + @Override + public void close() { + try { + if (readBuffer != null) { + readBuffer.close(); + currentPositionInStream = 0; + readBuffer = null; + } + if (writeBuffer != null) { + writeBuffer.flush(); + writeBuffer.close(); + writeBuffer = null; } } catch (IOException e) { throw new RuntimeException(e); diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java index b5189196e..e07225e05 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java @@ -1,65 +1,118 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; -import ru.mipt.java2016.homework.base.task2.KeyValueStorage; - import java.io.File; -import java.io.FileNotFoundException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ExecutionException; /** * Created by Lenovo on 31.10.2016. */ -public class ImplementationKeyValueStorage implements KeyValueStorage { +public class ImplementationKeyValueStorage implements ru.mipt.java2016.homework.base.task2.KeyValueStorage { - private final Map map = new HashMap<>(); + private final Map keyPositionMap = new HashMap<>(); private final SerialiserInterface keySerialiser; private final SerialiserInterface valueSerialiser; - private final FileWorker file; - private final String fileName; + private final Set deleteKeySet = new HashSet<>(); + + private final FileWorker keyPositionFile; + private final FileWorker valuesFile; + private final FileWorker deleteKeyFile; + private final FileWorker lockFile; + private final FileWorker validationFile; + + private final String dirPath; + + private Long currentPositionInValuesFile = new Long(0); + private final Integer lock = 42; + + private boolean writing = true; + private boolean needRebuild = false; private static final String VALIDATION_STRING = "EtotFileZapisanMoeiProgoi"; private boolean openFlag = true; + private LoadingCache cacheValues = CacheBuilder.newBuilder() + .maximumSize(10) + .build( + new CacheLoader() { + @Override + public V load(K k) throws StorageException { + V result = loadKey(keyPositionMap.get(k)); + if (result == null) { + throw new StorageException("no such key"); + } + return result; + } + }); - public ImplementationKeyValueStorage(String keyName, String valueName, + public ImplementationKeyValueStorage(//String keyName, String valueName, SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, String directoryPath) { this.keySerialiser = keySerialiser; this.valueSerialiser = valueSerialiser; - file = new FileWorker(); - fileName = directoryPath + File.separator + "myFile.db"; + keyPositionMap = Collections.synchronizedMap(new HashMap<>()); - try { - String inputData = file.read(fileName); - - String[] tokens = inputData.split("\n"); - if (!tokens[0].equals(VALIDATION_STRING)) { - throw new RuntimeException("Not my file"); - } - if (!tokens[1].equals(keyName)) { - throw new RuntimeException("Wrong key type"); - } - if (!tokens[2].equals(valueName)) { - throw new RuntimeException("Wrong value type"); + if (dirPath == null || dirPath.equals("")) { + this.dirPath = ""; + } else { + this.dirPath = dirPath + File.separator; + } + keyPositionFile = new FileWorker(this.dirPath + "keyPositionFile.db"); + valuesFile = new FileWorker(this.dirPath + "valuesFile.db"); + deleteKeyFile = new FileWorker(this.dirPath + "deleteKeyFile.db"); + validationFile = new FileWorker(this.dirPath + "validationFile.db"); + lockFile = new FileWorker(this.dirPath + "lockFile.db"); + if (lockFile.exist()) { + throw new RuntimeException("there is working storage"); + } else { + lockFile.createFile(); + } + if (!keyPositionFile.exist()) { + keyPositionFile.createFile(); + valuesFile.createFile(); + deleteKeyFile.createFile(); + validationFile.createFile(); + } else { + currentPositionInValuesFile = valuesFile.fileLength(); + initStorage(); + } + valuesFile.appendMode(); + } + private void initStorage() { + keyPositionFile.close(); + int cnt = 0; + String nextKey = keyPositionFile.read(); + while (nextKey != null) { + Long position = Long.parseLong(keyPositionFile.read()); + try { + keyPositionMap.put(keySerialiser.deserialise(nextKey), position); + } catch (StorageException e) { + throw new RuntimeException(e); } - - Integer objectsNumber = Integer.parseInt(tokens[3]); - for (int i = 0; i < objectsNumber; ++i) { - K key = keySerialiser.deserialise(tokens[2 * i + 4]); - V value = valueSerialiser.deserialise(tokens[2 * i + 5]); - map.put(key, value); + nextKey = keyPositionFile.read(); + cnt++; + } + nextKey = deleteKeyFile.read(); + while (nextKey != null) { + try { + K key = keySerialiser.deserialise(nextKey); + deleteKeySet.add(key); + keyPositionMap.remove(key); + } catch (StorageException e) { + throw new RuntimeException(e); } - - } catch (FileNotFoundException e) { - writeData(); + nextKey = deleteKeyFile.read(); + cnt++; + } + keyPositionFile.appendMode(); + deleteKeyFile.close(); + if (cnt > 2 * keyPositionMap.size()) { + needRebuild = true; } - - } private void checkIfFileIsOpen() { @@ -70,61 +123,146 @@ private void checkIfFileIsOpen() { @Override public V read(K key) { - checkIfFileIsOpen(); - return map.get(key); - + synchronized (lock) { + checkIfFileIsOpen(); + Long offset = keyPositionMap.get(key); + if (offset != null) { + try { + return cacheValues.get(key); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } else { + return null; + } + } } @Override public boolean exists(K key) { - checkIfFileIsOpen(); - return map.containsKey(key); + synchronized (lock) { + checkIfFileIsOpen(); + return keyPositionMap.containsKey(key); + } } @Override public void write(K key, V value) { - checkIfFileIsOpen(); - map.put(key, value); + synchronized (lock) { + checkIfFileIsOpen(); + deleteKeySet.remove(key); + keyPositionMap.put(key, currentPositionInValuesFile); + flush(key, value); + } } @Override public void delete(K key) { - checkIfFileIsOpen(); - map.remove(key); + synchronized (lock) { + checkIfFileIsOpen(); + deleteKeySet.add(key); + keyPositionMap.remove(key); + } } @Override public Iterator readKeys() { - checkIfFileIsOpen(); - return map.keySet().iterator(); + synchronized (lock) { + checkIfFileIsOpen(); + return keyPositionMap.keySet().iterator(); + } } - @Override public int size() { - checkIfFileIsOpen(); - return map.size(); + synchronized (lock) { + checkIfFileIsOpen(); + return keyPositionMap.size(); + } } @Override - public void close() { - checkIfFileIsOpen(); - openFlag = false; - writeData(); - map.clear(); + public void close() throws IOException { + synchronized (lock) { + if(openFlag) { + openFlag = false; + if (needRebuild) { + rebuild(); + } else { + flushDeletes(); + deleteKeyFile.close(); + keyPositionFile.close(); + valuesFile.close(); + writeChecksum(); + } + lockFile.delete(); + } + } + } + + private void rebuild() { + FileWorker newVal = new FileWorker(dirPath + "nva.db", false); + newVal.createFile(); + deleteKeyFile.close(); + deleteKeyFile.delete(); + deleteKeyFile.createFile(); + validationFile.close(); + validationFile.delete(); + validationFile.createFile(); + keyPositionFile.close(); + keyPositionFile.delete(); + keyPositionFile.createFile(); + currentPositionInValuesFile = 0L; + for (Map.Entry entry: keyPositionMap.entrySet()) { + keyPositionFile.write(keySerialiser.serialise(entry.getKey())); + keyPositionFile.write(currentPositionInValuesFile.toString()); + currentPositionInValuesFile += newVal.write(valueSerialiser.serialise(loadKey(entry.getValue()))); + } + keyPositionFile.writeSubmit(); + newVal.writeSubmit(); + writeChecksum(); + valuesFile.close(); + valuesFile.delete(); + newVal.rename(dirPath + "valuesFile.db"); + newVal.close(); } - public void writeData() { - StringBuffer text = new StringBuffer(VALIDATION_STRING + "\n"); - text.append(keySerialiser.getType()).append('\n').append(valueSerialiser.getType()).append('\n'); - text.append(map.size()).append('\n'); - for (Map.Entry entry : map.entrySet()) { - text.append(keySerialiser.serialise(entry.getKey())) - .append('\n') - .append(valueSerialiser.serialise(entry.getValue())) - .append('\n'); + private void writeChecksum() { + validationFile.close(); + validationFile.write(Long.toString(keyPositionFile.getCheckSum())); + validationFile.writeSubmit(); + } + private V loadKey(long position) { + if (writing) { + valuesFile.close(); + writing = false; + } + valuesFile.goToPosition(position); + try { + return valueSerialiser.deserialise( valuesFile.read()); + } catch (StorageException e) { + throw new RuntimeException(e); } - file.write(fileName, text.toString()); + } + private void flush(K key, V value) { + if (!writing) { + valuesFile.close(); + valuesFile.appendMode(); + writing = true; + } + keyPositionFile.write(keySerialiser.serialise(key)); + keyPositionFile.write(currentPositionInValuesFile.toString()); + currentPositionInValuesFile += valuesFile.write(valueSerialiser.serialise(value)); } + private void flushDeletes() { + if (!deleteKeyFile.exist()) { + deleteKeyFile.createFile(); + } + for (K entry : deleteKeySet) { + deleteKeyFile.write(keySerialiser.serialise(entry)); + } + deleteKeyFile.writeSubmit(); + deleteKeySet.clear(); + } } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java index 5bd639b6f..d32440a1f 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java @@ -1,28 +1,25 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; -import ru.mipt.java2016.homework.base.task2.KeyValueStorage; import ru.mipt.java2016.homework.tests.task2.AbstractSingleFileStorageTest; -import ru.mipt.java2016.homework.tests.task2.Student; -import ru.mipt.java2016.homework.tests.task2.StudentKey; public class ImplementationKeyValueStorageTest extends AbstractSingleFileStorageTest { @Override - protected KeyValueStorage buildStringsStorage(String path) { + protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildStringsStorage(String path) { return new ImplementationKeyValueStorage<>("String", "String", new SerialiseString(), new SerialiseString(), path) ; } @Override - protected KeyValueStorage buildNumbersStorage(String path) { + protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildNumbersStorage(String path) { return new ImplementationKeyValueStorage<>("Integer", "Double", new SerialiseInteger(), new SerialiseDouble(), path) ; } @Override - protected KeyValueStorage buildPojoStorage(String path) { + protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildPojoStorage(String path) { return new ImplementationKeyValueStorage<>("StudentKey", "Student", new SerialiseStudentKey(), new SerialiseStudent(), path) ; From e0d40ac8b8252a47be39e69d40ba5636d997821f Mon Sep 17 00:00:00 2001 From: Ksenia Proskurina Date: Sun, 18 Dec 2016 00:01:34 +0300 Subject: [PATCH 2/3] task 3 --- homework-g596-proskurina/pom.xml | 5 + .../g596/proskurina/task2/FileWorker.java | 4 +- .../task2/ImplementationKeyValueStorage.java | 143 ++++++++---------- .../ImplementationKeyValueStorageTest.java | 16 +- ...ntationKeyValueStoragePerformanceTest.java | 30 ++++ 5 files changed, 103 insertions(+), 95 deletions(-) create mode 100644 homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java diff --git a/homework-g596-proskurina/pom.xml b/homework-g596-proskurina/pom.xml index 2601fce37..3167acfd9 100644 --- a/homework-g596-proskurina/pom.xml +++ b/homework-g596-proskurina/pom.xml @@ -25,5 +25,10 @@ 1.0.0 test + + com.google.guava + guava + 19.0 + \ No newline at end of file diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java index c9defadf8..99c4467b3 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java @@ -6,7 +6,7 @@ /** * Created by Lenovo on 31.10.2016. */ -public class FileWorker implements Closeable{ +public class FileWorker implements Closeable { private final File file; private final String fileName; @@ -55,7 +55,7 @@ private boolean innerExist() throws FileNotFoundException { return true; } - public void writeSubmit() { + public void flushSubmit() { if (writeBuffer != null) { try { writeBuffer.flush(); diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java index e07225e05..b42dc290c 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java @@ -1,5 +1,9 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + import java.io.File; import java.io.IOException; import java.util.*; @@ -10,7 +14,7 @@ */ public class ImplementationKeyValueStorage implements ru.mipt.java2016.homework.base.task2.KeyValueStorage { - private final Map keyPositionMap = new HashMap<>(); + private final Map keyPositionMap; private final SerialiserInterface keySerialiser; private final SerialiserInterface valueSerialiser; @@ -21,53 +25,49 @@ public class ImplementationKeyValueStorage implements ru.mipt.java2016.hom private final FileWorker valuesFile; private final FileWorker deleteKeyFile; private final FileWorker lockFile; - private final FileWorker validationFile; - private final String dirPath; + private final String directoryPath; private Long currentPositionInValuesFile = new Long(0); private final Integer lock = 42; private boolean writing = true; private boolean needRebuild = false; - private static final String VALIDATION_STRING = "EtotFileZapisanMoeiProgoi"; private boolean openFlag = true; private LoadingCache cacheValues = CacheBuilder.newBuilder() - .maximumSize(10) + .maximumSize(42) .build( new CacheLoader() { @Override - public V load(K k) throws StorageException { - V result = loadKey(keyPositionMap.get(k)); + public V load(K k) throws RuntimeException { + V result = readKey(keyPositionMap.get(k)); if (result == null) { - throw new StorageException("no such key"); + throw new RuntimeException("no such key"); } return result; } }); - public ImplementationKeyValueStorage(//String keyName, String valueName, - SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, + public ImplementationKeyValueStorage(SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, String directoryPath) { this.keySerialiser = keySerialiser; this.valueSerialiser = valueSerialiser; - keyPositionMap = Collections.synchronizedMap(new HashMap<>()); + keyPositionMap = new HashMap<>(); - if (dirPath == null || dirPath.equals("")) { - this.dirPath = ""; + if (directoryPath == null || directoryPath.equals("")) { + this.directoryPath = ""; } else { - this.dirPath = dirPath + File.separator; + this.directoryPath = directoryPath + File.separator; } - keyPositionFile = new FileWorker(this.dirPath + "keyPositionFile.db"); - valuesFile = new FileWorker(this.dirPath + "valuesFile.db"); - deleteKeyFile = new FileWorker(this.dirPath + "deleteKeyFile.db"); - validationFile = new FileWorker(this.dirPath + "validationFile.db"); - lockFile = new FileWorker(this.dirPath + "lockFile.db"); + keyPositionFile = new FileWorker(this.directoryPath + "keyPositionFile.db"); + valuesFile = new FileWorker(this.directoryPath + "valuesFile.db"); + deleteKeyFile = new FileWorker(this.directoryPath + "deleteKeyFile.db"); + lockFile = new FileWorker(this.directoryPath + "lockFile.db"); if (lockFile.exist()) { - throw new RuntimeException("there is working storage"); + throw new RuntimeException("we already have working storage"); } else { lockFile.createFile(); } @@ -75,36 +75,28 @@ public ImplementationKeyValueStorage(//String keyName, String valueName, keyPositionFile.createFile(); valuesFile.createFile(); deleteKeyFile.createFile(); - validationFile.createFile(); } else { currentPositionInValuesFile = valuesFile.fileLength(); initStorage(); } - valuesFile.appendMode(); + valuesFile.appendMode(); } + private void initStorage() { keyPositionFile.close(); int cnt = 0; String nextKey = keyPositionFile.read(); while (nextKey != null) { Long position = Long.parseLong(keyPositionFile.read()); - try { - keyPositionMap.put(keySerialiser.deserialise(nextKey), position); - } catch (StorageException e) { - throw new RuntimeException(e); - } + keyPositionMap.put(keySerialiser.deserialise(nextKey), position); nextKey = keyPositionFile.read(); cnt++; } nextKey = deleteKeyFile.read(); while (nextKey != null) { - try { - K key = keySerialiser.deserialise(nextKey); - deleteKeySet.add(key); - keyPositionMap.remove(key); - } catch (StorageException e) { - throw new RuntimeException(e); - } + K key = keySerialiser.deserialise(nextKey); + deleteKeySet.add(key); + keyPositionMap.remove(key); nextKey = deleteKeyFile.read(); cnt++; } @@ -115,7 +107,7 @@ private void initStorage() { } } - private void checkIfFileIsOpen() { + private void checkIfStorageIsOpen() { if (!openFlag) { throw new RuntimeException("Storage is closed"); } @@ -124,9 +116,9 @@ private void checkIfFileIsOpen() { @Override public V read(K key) { synchronized (lock) { - checkIfFileIsOpen(); - Long offset = keyPositionMap.get(key); - if (offset != null) { + checkIfStorageIsOpen(); + Long position = keyPositionMap.get(key); + if (position != null) { try { return cacheValues.get(key); } catch (ExecutionException e) { @@ -141,7 +133,7 @@ public V read(K key) { @Override public boolean exists(K key) { synchronized (lock) { - checkIfFileIsOpen(); + checkIfStorageIsOpen(); return keyPositionMap.containsKey(key); } } @@ -149,17 +141,17 @@ public boolean exists(K key) { @Override public void write(K key, V value) { synchronized (lock) { - checkIfFileIsOpen(); + checkIfStorageIsOpen(); deleteKeySet.remove(key); keyPositionMap.put(key, currentPositionInValuesFile); - flush(key, value); + writeToFile(key, value); } } @Override public void delete(K key) { synchronized (lock) { - checkIfFileIsOpen(); + checkIfStorageIsOpen(); deleteKeySet.add(key); keyPositionMap.remove(key); } @@ -168,14 +160,15 @@ public void delete(K key) { @Override public Iterator readKeys() { synchronized (lock) { - checkIfFileIsOpen(); + checkIfStorageIsOpen(); return keyPositionMap.keySet().iterator(); } } + @Override public int size() { synchronized (lock) { - checkIfFileIsOpen(); + checkIfStorageIsOpen(); return keyPositionMap.size(); } } @@ -183,16 +176,15 @@ public int size() { @Override public void close() throws IOException { synchronized (lock) { - if(openFlag) { + if (openFlag) { openFlag = false; if (needRebuild) { rebuild(); } else { - flushDeletes(); + writeToFileDeleteKeySet(); deleteKeyFile.close(); keyPositionFile.close(); - valuesFile.close(); - writeChecksum(); + valuesFile.close(); } lockFile.delete(); } @@ -200,54 +192,37 @@ public void close() throws IOException { } private void rebuild() { - FileWorker newVal = new FileWorker(dirPath + "nva.db", false); - newVal.createFile(); + FileWorker newValuesFile = new FileWorker(directoryPath + "newValuesFile.db"); + newValuesFile.createFile(); deleteKeyFile.close(); - deleteKeyFile.delete(); - deleteKeyFile.createFile(); - validationFile.close(); - validationFile.delete(); - validationFile.createFile(); keyPositionFile.close(); - keyPositionFile.delete(); - keyPositionFile.createFile(); currentPositionInValuesFile = 0L; for (Map.Entry entry: keyPositionMap.entrySet()) { keyPositionFile.write(keySerialiser.serialise(entry.getKey())); keyPositionFile.write(currentPositionInValuesFile.toString()); - currentPositionInValuesFile += newVal.write(valueSerialiser.serialise(loadKey(entry.getValue()))); - } - keyPositionFile.writeSubmit(); - newVal.writeSubmit(); - writeChecksum(); - valuesFile.close(); - valuesFile.delete(); - newVal.rename(dirPath + "valuesFile.db"); - newVal.close(); + currentPositionInValuesFile += newValuesFile.write(valueSerialiser.serialise(readKey(entry.getValue()))); + } + keyPositionFile.flushSubmit(); + newValuesFile.flushSubmit(); + valuesFile.close(); + valuesFile.delete(); + newValuesFile.rename(directoryPath + "valuesFile.db"); + newValuesFile.close(); } - private void writeChecksum() { - validationFile.close(); - validationFile.write(Long.toString(keyPositionFile.getCheckSum())); - validationFile.writeSubmit(); - } - private V loadKey(long position) { + private V readKey(long position) { if (writing) { - valuesFile.close(); + valuesFile.close(); writing = false; } - valuesFile.goToPosition(position); - try { - return valueSerialiser.deserialise( valuesFile.read()); - } catch (StorageException e) { - throw new RuntimeException(e); - } + valuesFile.goToPosition(position); + return valueSerialiser.deserialise(valuesFile.read()); } - private void flush(K key, V value) { + private void writeToFile(K key, V value) { if (!writing) { - valuesFile.close(); - valuesFile.appendMode(); + valuesFile.close(); + valuesFile.appendMode(); writing = true; } keyPositionFile.write(keySerialiser.serialise(key)); @@ -255,14 +230,14 @@ private void flush(K key, V value) { currentPositionInValuesFile += valuesFile.write(valueSerialiser.serialise(value)); } - private void flushDeletes() { + private void writeToFileDeleteKeySet() { if (!deleteKeyFile.exist()) { deleteKeyFile.createFile(); } for (K entry : deleteKeySet) { deleteKeyFile.write(keySerialiser.serialise(entry)); } - deleteKeyFile.writeSubmit(); + deleteKeyFile.flushSubmit(); deleteKeySet.clear(); } } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java index d32440a1f..7c6ab4bf2 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java @@ -1,27 +1,25 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; +import ru.mipt.java2016.homework.base.task2.KeyValueStorage; import ru.mipt.java2016.homework.tests.task2.AbstractSingleFileStorageTest; public class ImplementationKeyValueStorageTest extends AbstractSingleFileStorageTest { @Override - protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildStringsStorage(String path) { - return new ImplementationKeyValueStorage<>("String", "String", - new SerialiseString(), new SerialiseString(), + protected KeyValueStorage buildStringsStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseString(), new SerialiseString(), path) ; } @Override - protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildNumbersStorage(String path) { - return new ImplementationKeyValueStorage<>("Integer", "Double", - new SerialiseInteger(), new SerialiseDouble(), + protected KeyValueStorage buildNumbersStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseInteger(), new SerialiseDouble(), path) ; } @Override - protected ru.mipt.java2016.homework.base.task2.KeyValueStorage buildPojoStorage(String path) { - return new ImplementationKeyValueStorage<>("StudentKey", "Student", - new SerialiseStudentKey(), new SerialiseStudent(), + protected KeyValueStorage buildPojoStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseStudentKey(), new SerialiseStudent(), path) ; } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java new file mode 100644 index 000000000..c6ba9b3c4 --- /dev/null +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java @@ -0,0 +1,30 @@ +package ru.mipt.java2016.homework.g596.proskurina.task3; + +import ru.mipt.java2016.homework.base.task2.KeyValueStorage; +import ru.mipt.java2016.homework.g596.proskurina.task2.*; +import ru.mipt.java2016.homework.tests.task3.KeyValueStoragePerformanceTest; + +/** + * Created by Lenovo on 17.12.2016. + */ +public class ImplementationKeyValueStoragePerformanceTest extends KeyValueStoragePerformanceTest { + + @Override + protected KeyValueStorage buildStringsStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseString(), new SerialiseString(), + path) ; + } + + @Override + protected KeyValueStorage buildNumbersStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseInteger(), new SerialiseDouble(), + path) ; + } + + @Override + protected KeyValueStorage buildPojoStorage(String path) { + return new ImplementationKeyValueStorage<>(new SerialiseStudentKey(), new SerialiseStudent(), + path) ; + } + +} From af0bd6705fe83f03b0307481cbaa8d59ff9cb4a5 Mon Sep 17 00:00:00 2001 From: Ksenia Proskurina Date: Tue, 20 Dec 2016 13:33:11 +0300 Subject: [PATCH 3/3] task3 fixes --- .../g596/proskurina/task2/FileWorker.java | 166 ++--------- .../task2/ImplementationKeyValueStorage.java | 257 +++++------------ .../proskurina/task2/SerialiseDouble.java | 4 - .../proskurina/task2/SerialiseInteger.java | 4 - .../proskurina/task2/SerialiseString.java | 4 - .../proskurina/task2/SerialiserInterface.java | 1 - .../g596/proskurina/task3/FileWorker.java | 167 +++++++++++ .../task3/ImplementationKeyValueStorage.java | 264 ++++++++++++++++++ .../ImplementationKeyValueStorageTest.java | 9 +- .../proskurina/task2/SerialiseStudent.java | 4 - .../proskurina/task2/SerialiseStudentKey.java | 5 - ...ntationKeyValueStoragePerformanceTest.java | 2 +- 12 files changed, 540 insertions(+), 347 deletions(-) create mode 100644 homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/FileWorker.java create mode 100644 homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStorage.java diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java index 99c4467b3..5bc16778f 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/FileWorker.java @@ -1,168 +1,60 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; import java.io.*; -import java.nio.ByteBuffer; /** * Created by Lenovo on 31.10.2016. */ -public class FileWorker implements Closeable { +public class FileWorker { - private final File file; - private final String fileName; - - private InputStream readBuffer = null; - private OutputStream writeBuffer = null; - - private long currentPositionInStream = 0; - - public FileWorker(String fileName) { - this.file = new File(fileName); - this.fileName = fileName; - } - - public void createFile() { - try { - file.createNewFile(); - } catch (IOException e) { - System.out.println("file didn't created"); - throw new RuntimeException(e); - } - } - - public void rename(String newName) { - File newFile = new File(newName); - file.renameTo(newFile); - } - - public void appendMode() { - try { - close(); - writeBuffer = new FileOutputStream(file.getAbsoluteFile(), true); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - public boolean exist() { - return file.exists(); - } - - private boolean innerExist() throws FileNotFoundException { - if (!exist()) { - throw new FileNotFoundException(fileName); - } - return true; - } - - public void flushSubmit() { - if (writeBuffer != null) { - try { - writeBuffer.flush(); - writeBuffer.close(); - writeBuffer = null; - } catch (IOException e) { - throw new RuntimeException(e); - } + private static void exists(String fileName) throws FileNotFoundException { + File file = new File(fileName); + if (!file.exists()) { + throw new FileNotFoundException(file.getName()); } } - public long write(String str) { - try { - innerExist(); - if (writeBuffer == null) { - writeBuffer = new BufferedOutputStream(new FileOutputStream(file.getAbsoluteFile())); - } - byte[] bytes = str.getBytes(); - byte[] bytesNumber = ByteBuffer.allocate(4).putInt(bytes.length).array(); - writeBuffer.write(bytesNumber); - writeBuffer.write(bytes); - return 4 + bytes.length; - } catch (IOException e) { - throw new RuntimeException(e); - } + public FileWorker() { } - public String read() { - try { - innerExist(); - if (readBuffer == null) { - readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); - currentPositionInStream = 0; - } - if (readBuffer.available() < 4) { - readBuffer.close(); - readBuffer = null; - return null; - } - byte[] bytesNumberArray = new byte[4]; - int bytesNumberArraySize = readBuffer.read(bytesNumberArray, 0, 4); - //addToCalc(bytes); - if (bytesNumberArraySize < 4) { - readBuffer.close(); - throw new RuntimeException("Error in reading bytes number"); - } - currentPositionInStream += bytesNumberArraySize; - int bytesNumber = ByteBuffer.wrap(bytesNumberArray).getInt(); - byte[] bytesArray = new byte[bytesNumber]; - int bytesArraySize = readBuffer.read(bytesArray, 0, bytesNumber); - //addToCalc(bytes); - if (bytesArraySize < bytesNumber) { - readBuffer.close(); - throw new RuntimeException("Error in reading bytes"); - } - currentPositionInStream += bytesArraySize; - return new String(bytesArray); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + public static String read(String fileName) throws FileNotFoundException { - public long fileLength() { - try (FileInputStream inputStream = new FileInputStream(file.getAbsoluteFile())) { - return inputStream.available(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } + StringBuffer inputData = new StringBuffer(); + exists(fileName); + File file = new File(fileName); - public void goToPosition(long position) { try { - innerExist(); - if (readBuffer == null || currentPositionInStream > position) { - if (readBuffer != null) { - readBuffer.close(); + BufferedReader in = new BufferedReader(new FileReader(file.getAbsoluteFile())); + try { + String s; + while ((s = in.readLine()) != null) { + inputData.append(s); + inputData.append("\n"); } - readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); - currentPositionInStream = 0; - } - while (currentPositionInStream < position) { - currentPositionInStream += readBuffer.skip(position - currentPositionInStream); + } finally { + in.close(); } } catch (IOException e) { throw new RuntimeException(e); } + return inputData.toString(); } - public void delete() { - file.delete(); - } + public static void write(String fileName, String text) { - @Override - public void close() { + File file = new File(fileName); try { - if (readBuffer != null) { - readBuffer.close(); - currentPositionInStream = 0; - readBuffer = null; + if (!file.exists()) { + file.createNewFile(); } - if (writeBuffer != null) { - writeBuffer.flush(); - writeBuffer.close(); - writeBuffer = null; + PrintWriter out = new PrintWriter(file.getAbsoluteFile()); + try { + out.print(text); + } finally { + out.close(); } } catch (IOException e) { throw new RuntimeException(e); } } -} +} \ No newline at end of file diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java index b42dc290c..fac6b4e31 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorage.java @@ -1,243 +1,132 @@ package ru.mipt.java2016.homework.g596.proskurina.task2; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import ru.mipt.java2016.homework.base.task2.KeyValueStorage; -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ExecutionException; +import java.io.FileNotFoundException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; /** * Created by Lenovo on 31.10.2016. */ -public class ImplementationKeyValueStorage implements ru.mipt.java2016.homework.base.task2.KeyValueStorage { +public class ImplementationKeyValueStorage implements KeyValueStorage { - private final Map keyPositionMap; + private final HashMap map = new HashMap<>(); + + private final String keyName; + private final String valueName; private final SerialiserInterface keySerialiser; private final SerialiserInterface valueSerialiser; - private final Set deleteKeySet = new HashSet<>(); - - private final FileWorker keyPositionFile; - private final FileWorker valuesFile; - private final FileWorker deleteKeyFile; - private final FileWorker lockFile; - - private final String directoryPath; - - private Long currentPositionInValuesFile = new Long(0); - private final Integer lock = 42; - - private boolean writing = true; - private boolean needRebuild = false; + private final FileWorker file; + private final String fileName; + private static final String VALIDATION_STRING = "EtotFileZapisanMoeiProgoi"; private boolean openFlag = true; - private LoadingCache cacheValues = CacheBuilder.newBuilder() - .maximumSize(42) - .build( - new CacheLoader() { - @Override - public V load(K k) throws RuntimeException { - V result = readKey(keyPositionMap.get(k)); - if (result == null) { - throw new RuntimeException("no such key"); - } - return result; - } - }); - - public ImplementationKeyValueStorage(SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, + + public ImplementationKeyValueStorage(String keyName, String valueName, + SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, String directoryPath) { + this.keyName = keyName; + this.valueName = valueName; + this.keySerialiser = keySerialiser; this.valueSerialiser = valueSerialiser; - keyPositionMap = new HashMap<>(); + file = new FileWorker(); + fileName = directoryPath + "/myFile.db"; - if (directoryPath == null || directoryPath.equals("")) { - this.directoryPath = ""; - } else { - this.directoryPath = directoryPath + File.separator; - } - keyPositionFile = new FileWorker(this.directoryPath + "keyPositionFile.db"); - valuesFile = new FileWorker(this.directoryPath + "valuesFile.db"); - deleteKeyFile = new FileWorker(this.directoryPath + "deleteKeyFile.db"); - lockFile = new FileWorker(this.directoryPath + "lockFile.db"); - if (lockFile.exist()) { - throw new RuntimeException("we already have working storage"); - } else { - lockFile.createFile(); - } - if (!keyPositionFile.exist()) { - keyPositionFile.createFile(); - valuesFile.createFile(); - deleteKeyFile.createFile(); - } else { - currentPositionInValuesFile = valuesFile.fileLength(); - initStorage(); - } - valuesFile.appendMode(); - } + try { + String inputData = file.read(fileName); - private void initStorage() { - keyPositionFile.close(); - int cnt = 0; - String nextKey = keyPositionFile.read(); - while (nextKey != null) { - Long position = Long.parseLong(keyPositionFile.read()); - keyPositionMap.put(keySerialiser.deserialise(nextKey), position); - nextKey = keyPositionFile.read(); - cnt++; - } - nextKey = deleteKeyFile.read(); - while (nextKey != null) { - K key = keySerialiser.deserialise(nextKey); - deleteKeySet.add(key); - keyPositionMap.remove(key); - nextKey = deleteKeyFile.read(); - cnt++; - } - keyPositionFile.appendMode(); - deleteKeyFile.close(); - if (cnt > 2 * keyPositionMap.size()) { - needRebuild = true; - } - } + String[] tokens = inputData.split("\n"); + if (!tokens[0].equals(VALIDATION_STRING)) { + throw new RuntimeException("Not my file"); + } + if (!tokens[1].equals(keyName)) { + throw new RuntimeException("Wrong key type"); + } + if (!tokens[2].equals(valueName)) { + throw new RuntimeException("Wrong value type"); + } - private void checkIfStorageIsOpen() { - if (!openFlag) { - throw new RuntimeException("Storage is closed"); + Integer objectsNumber = Integer.parseInt(tokens[3]); + for (int i = 0; i < objectsNumber; ++i) { + K key = keySerialiser.deserialise(tokens[2 * i + 4]); + V value = valueSerialiser.deserialise(tokens[2 * i + 5]); + map.put(key, value); + } + + } catch (FileNotFoundException e) { + writeData(); } + + } @Override public V read(K key) { - synchronized (lock) { - checkIfStorageIsOpen(); - Long position = keyPositionMap.get(key); - if (position != null) { - try { - return cacheValues.get(key); - } catch (ExecutionException e) { - throw new RuntimeException(e); - } - } else { - return null; - } + if (openFlag) { + return map.get(key); + } else { + throw new RuntimeException("Storage is closed"); } } @Override public boolean exists(K key) { - synchronized (lock) { - checkIfStorageIsOpen(); - return keyPositionMap.containsKey(key); - } + return map.containsKey(key); } @Override public void write(K key, V value) { - synchronized (lock) { - checkIfStorageIsOpen(); - deleteKeySet.remove(key); - keyPositionMap.put(key, currentPositionInValuesFile); - writeToFile(key, value); + if (openFlag) { + map.put(key, value); + } else { + throw new RuntimeException("Storage is closed"); } } @Override public void delete(K key) { - synchronized (lock) { - checkIfStorageIsOpen(); - deleteKeySet.add(key); - keyPositionMap.remove(key); - } + map.remove(key); } @Override public Iterator readKeys() { - synchronized (lock) { - checkIfStorageIsOpen(); - return keyPositionMap.keySet().iterator(); + if (openFlag) { + return map.keySet().iterator(); + } else { + throw new RuntimeException("Storage is closed"); } } @Override public int size() { - synchronized (lock) { - checkIfStorageIsOpen(); - return keyPositionMap.size(); - } + return map.size(); } @Override - public void close() throws IOException { - synchronized (lock) { - if (openFlag) { - openFlag = false; - if (needRebuild) { - rebuild(); - } else { - writeToFileDeleteKeySet(); - deleteKeyFile.close(); - keyPositionFile.close(); - valuesFile.close(); - } - lockFile.delete(); - } - } + public void close() { + openFlag = false; + writeData(); } - private void rebuild() { - FileWorker newValuesFile = new FileWorker(directoryPath + "newValuesFile.db"); - newValuesFile.createFile(); - deleteKeyFile.close(); - keyPositionFile.close(); - currentPositionInValuesFile = 0L; - for (Map.Entry entry: keyPositionMap.entrySet()) { - keyPositionFile.write(keySerialiser.serialise(entry.getKey())); - keyPositionFile.write(currentPositionInValuesFile.toString()); - currentPositionInValuesFile += newValuesFile.write(valueSerialiser.serialise(readKey(entry.getValue()))); + public void writeData() { + StringBuffer text = new StringBuffer(VALIDATION_STRING + "\n"); + text.append(keyName).append('\n').append(valueName).append('\n'); + text.append(map.size()).append('\n'); + for (Map.Entry entry : map.entrySet()) { + text.append(keySerialiser.serialise(entry.getKey())) + .append('\n') + .append(valueSerialiser.serialise(entry.getValue())) + .append('\n'); } - keyPositionFile.flushSubmit(); - newValuesFile.flushSubmit(); - valuesFile.close(); - valuesFile.delete(); - newValuesFile.rename(directoryPath + "valuesFile.db"); - newValuesFile.close(); - } + file.write(fileName, text.toString()); - private V readKey(long position) { - if (writing) { - valuesFile.close(); - writing = false; - } - valuesFile.goToPosition(position); - return valueSerialiser.deserialise(valuesFile.read()); } - private void writeToFile(K key, V value) { - if (!writing) { - valuesFile.close(); - valuesFile.appendMode(); - writing = true; - } - keyPositionFile.write(keySerialiser.serialise(key)); - keyPositionFile.write(currentPositionInValuesFile.toString()); - currentPositionInValuesFile += valuesFile.write(valueSerialiser.serialise(value)); - } - - private void writeToFileDeleteKeySet() { - if (!deleteKeyFile.exist()) { - deleteKeyFile.createFile(); - } - for (K entry : deleteKeySet) { - deleteKeyFile.write(keySerialiser.serialise(entry)); - } - deleteKeyFile.flushSubmit(); - deleteKeySet.clear(); - } -} +} \ No newline at end of file diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseDouble.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseDouble.java index 35015e12e..5b6613a34 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseDouble.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseDouble.java @@ -14,8 +14,4 @@ public Double deserialise(String inString) { return Double.parseDouble(inString); } - @Override - public String getType() { - return "Double"; - } } diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseInteger.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseInteger.java index 2d58a183b..1b92d7f95 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseInteger.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseInteger.java @@ -15,8 +15,4 @@ public Integer deserialise(String inString) { return Integer.parseInt(inString); } - @Override - public String getType() { - return "Integer"; - } } diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseString.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseString.java index 3ca159ecd..f51b6d343 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseString.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseString.java @@ -16,8 +16,4 @@ public String deserialise(String inString) { return inString; } - @Override - public String getType() { - return "String"; - } } diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiserInterface.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiserInterface.java index c92800c09..e26ace3b3 100644 --- a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiserInterface.java +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiserInterface.java @@ -9,5 +9,4 @@ public interface SerialiserInterface { T deserialise(String inString); - String getType(); } diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/FileWorker.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/FileWorker.java new file mode 100644 index 000000000..182704eeb --- /dev/null +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/FileWorker.java @@ -0,0 +1,167 @@ +package ru.mipt.java2016.homework.g596.proskurina.task3; + +import java.io.*; +import java.nio.ByteBuffer; + +/** + * Created by Lenovo on 31.10.2016. + */ +public class FileWorker implements Closeable { + + private final File file; + private final String fileName; + + private InputStream readBuffer; + private OutputStream writeBuffer; + + private long currentPositionInStream; + + public FileWorker(String fileName) { + this.file = new File(fileName); + this.fileName = fileName; + } + + public void createFile() { + try { + file.createNewFile(); + } catch (IOException e) { + System.out.println("file didn't created"); + throw new RuntimeException(e); + } + } + + public void rename(String newName) { + File newFile = new File(newName); + file.renameTo(newFile); + } + + public void appendMode() { + try { + close(); + writeBuffer = new FileOutputStream(file.getAbsoluteFile(), true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public boolean exist() { + return file.exists(); + } + + private boolean innerExist() throws FileNotFoundException { + if (!exist()) { + throw new FileNotFoundException(fileName); + } + return true; + } + + public void flushSubmit() { + if (writeBuffer != null) { + try { + writeBuffer.flush(); + writeBuffer.close(); + writeBuffer = null; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public long write(String str) { + try { + innerExist(); + if (writeBuffer == null) { + writeBuffer = new BufferedOutputStream(new FileOutputStream(file.getAbsoluteFile())); + } + byte[] bytes = str.getBytes(); + byte[] bytesNumber = ByteBuffer.allocate(4).putInt(bytes.length).array(); + writeBuffer.write(bytesNumber); + writeBuffer.write(bytes); + return 4 + bytes.length; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public String read() { + try { + innerExist(); + if (readBuffer == null) { + readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); + currentPositionInStream = 0; + } + if (readBuffer.available() < 4) { + readBuffer.close(); + readBuffer = null; + return null; + } + byte[] bytesNumberArray = new byte[4]; + int bytesNumberArraySize = readBuffer.read(bytesNumberArray, 0, 4); + //addToCalc(bytes); + if (bytesNumberArraySize < 4) { + readBuffer.close(); + throw new RuntimeException("Error in reading bytes number"); + } + currentPositionInStream += bytesNumberArraySize; + int bytesNumber = ByteBuffer.wrap(bytesNumberArray).getInt(); + byte[] bytesArray = new byte[bytesNumber]; + int bytesArraySize = readBuffer.read(bytesArray, 0, bytesNumber); + if (bytesArraySize < bytesNumber) { + readBuffer.close(); + throw new RuntimeException("Error in reading bytes"); + } + currentPositionInStream += bytesArraySize; + return new String(bytesArray); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public long fileLength() { + try (FileInputStream inputStream = new FileInputStream(file.getAbsoluteFile())) { + return inputStream.available(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void goToPosition(long position) { + try { + innerExist(); + if (readBuffer == null || currentPositionInStream > position) { + if (readBuffer != null) { + readBuffer.close(); + } + readBuffer = new BufferedInputStream(new FileInputStream(file.getAbsoluteFile())); + currentPositionInStream = 0; + } + while (currentPositionInStream < position) { + currentPositionInStream += readBuffer.skip(position - currentPositionInStream); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void delete() { + file.delete(); + } + + @Override + public void close() { + try { + if (readBuffer != null) { + readBuffer.close(); + currentPositionInStream = 0; + readBuffer = null; + } + if (writeBuffer != null) { + writeBuffer.flush(); + writeBuffer.close(); + writeBuffer = null; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStorage.java b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStorage.java new file mode 100644 index 000000000..006e8bd68 --- /dev/null +++ b/homework-g596-proskurina/src/main/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStorage.java @@ -0,0 +1,264 @@ +package ru.mipt.java2016.homework.g596.proskurina.task3; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import ru.mipt.java2016.homework.base.task2.KeyValueStorage; +import ru.mipt.java2016.homework.g596.proskurina.task2.SerialiserInterface; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Created by Lenovo on 31.10.2016. + */ +public class ImplementationKeyValueStorage implements KeyValueStorage { + + private final Map keyPositionMap; + + private final SerialiserInterface keySerialiser; + private final SerialiserInterface valueSerialiser; + + private final Set deleteKeySet = new HashSet<>(); + + private final FileWorker keyPositionFile; + private final FileWorker valuesFile; + private final FileWorker deleteKeyFile; + private final FileWorker lockFile; + + private final String directoryPath; + + private Long currentPositionInValuesFile = new Long(0); + + private final ReentrantReadWriteLock rwlock = new ReentrantReadWriteLock(); + private final Lock rlock = rwlock.readLock(); + private final Lock wlock = rwlock.writeLock(); + + private boolean writing = true; + private boolean needRebuild = false; + private boolean openFlag = true; + + private final int maxSizeOfCache = 42; + + private LoadingCache cacheValues = CacheBuilder.newBuilder() + .maximumSize(maxSizeOfCache) + .build( + new CacheLoader() { + @Override + public V load(K k) throws RuntimeException { + V result = readKey(keyPositionMap.get(k)); + if (result == null) { + throw new RuntimeException("no such key"); + } + return result; + } + }); + + public ImplementationKeyValueStorage(SerialiserInterface keySerialiser, SerialiserInterface valueSerialiser, + String directoryPath) { + + this.keySerialiser = keySerialiser; + this.valueSerialiser = valueSerialiser; + + keyPositionMap = new HashMap<>(); + + if (directoryPath == null || directoryPath.equals("")) { + this.directoryPath = ""; + } else { + this.directoryPath = directoryPath + File.separator; + } + keyPositionFile = new FileWorker(this.directoryPath + "keyPositionFile.db"); + valuesFile = new FileWorker(this.directoryPath + "valuesFile.db"); + deleteKeyFile = new FileWorker(this.directoryPath + "deleteKeyFile.db"); + lockFile = new FileWorker(this.directoryPath + "lockFile.db"); + if (lockFile.exist()) { + throw new RuntimeException("we already have working storage"); + } else { + lockFile.createFile(); + } + if (!keyPositionFile.exist()) { + keyPositionFile.createFile(); + valuesFile.createFile(); + deleteKeyFile.createFile(); + } else { + currentPositionInValuesFile = valuesFile.fileLength(); + initStorage(); + } + valuesFile.appendMode(); + } + + private void initStorage() { + keyPositionFile.close(); + int cnt = 0; + String nextKey = keyPositionFile.read(); + while (nextKey != null) { + Long position = Long.parseLong(keyPositionFile.read()); + keyPositionMap.put(keySerialiser.deserialise(nextKey), position); + nextKey = keyPositionFile.read(); + cnt++; + } + nextKey = deleteKeyFile.read(); + while (nextKey != null) { + K key = keySerialiser.deserialise(nextKey); + deleteKeySet.add(key); + keyPositionMap.remove(key); + nextKey = deleteKeyFile.read(); + cnt++; + } + keyPositionFile.appendMode(); + deleteKeyFile.close(); + if (cnt > 2 * keyPositionMap.size()) { + needRebuild = true; + } + } + + private void checkIfStorageIsOpen() { + if (!openFlag) { + throw new RuntimeException("Storage is closed"); + } + } + + @Override + public V read(K key) { + wlock.lock(); + try { + checkIfStorageIsOpen(); + Long position = keyPositionMap.get(key); + if (position != null) { + try { + return cacheValues.get(key); + } catch (ExecutionException e) { + throw new RuntimeException(e); + } + } else { + return null; + } + } finally { + wlock.unlock(); + } + } + + @Override + public boolean exists(K key) { + wlock.lock(); + try { + checkIfStorageIsOpen(); + return keyPositionMap.containsKey(key); + } finally { + wlock.unlock(); + } + } + + + @Override + public void write(K key, V value) { + wlock.lock(); + checkIfStorageIsOpen(); + deleteKeySet.remove(key); + keyPositionMap.put(key, currentPositionInValuesFile); + writeToFile(key, value); + wlock.unlock(); + } + + @Override + public void delete(K key) { + wlock.lock(); + checkIfStorageIsOpen(); + deleteKeySet.add(key); + keyPositionMap.remove(key); + wlock.unlock(); + } + + @Override + public Iterator readKeys() { + wlock.lock(); + try { + checkIfStorageIsOpen(); + return keyPositionMap.keySet().iterator(); + } finally { + wlock.unlock(); + } + } + + @Override + public int size() { + rlock.lock(); + try { + checkIfStorageIsOpen(); + return keyPositionMap.size(); + } finally { + rlock.unlock(); + } + } + + @Override + public void close() throws IOException { + wlock.lock(); + if (openFlag) { + openFlag = false; + keyPositionFile.close(); + if (needRebuild) { + rebuild(); + } else { + writeToFileDeleteKeySet(); + valuesFile.close(); + } + deleteKeyFile.close(); + lockFile.delete(); + } + wlock.unlock(); + } + + private void rebuild() { + try (FileWorker newValuesFile = new FileWorker(directoryPath + "newValuesFile.db")) { + newValuesFile.createFile(); + currentPositionInValuesFile = 0L; + for (Map.Entry entry : keyPositionMap.entrySet()) { + keyPositionFile.write(keySerialiser.serialise(entry.getKey())); + keyPositionFile.write(currentPositionInValuesFile.toString()); + currentPositionInValuesFile += + newValuesFile.write(valueSerialiser.serialise(readKey(entry.getValue()))); + } + keyPositionFile.flushSubmit(); + newValuesFile.flushSubmit(); + valuesFile.close(); + valuesFile.delete(); + newValuesFile.rename(directoryPath + "valuesFile.db"); + } + } + + private V readKey(long position) { + if (writing) { + valuesFile.close(); + writing = false; + } + valuesFile.goToPosition(position); + return valueSerialiser.deserialise(valuesFile.read()); + } + + private void writeToFile(K key, V value) { + if (!writing) { + valuesFile.close(); + valuesFile.appendMode(); + writing = true; + } + keyPositionFile.write(keySerialiser.serialise(key)); + keyPositionFile.write(currentPositionInValuesFile.toString()); + currentPositionInValuesFile += valuesFile.write(valueSerialiser.serialise(value)); + } + + private void writeToFileDeleteKeySet() { + if (!deleteKeyFile.exist()) { + deleteKeyFile.createFile(); + } + for (K entry : deleteKeySet) { + deleteKeyFile.write(keySerialiser.serialise(entry)); + } + deleteKeyFile.flushSubmit(); + deleteKeySet.clear(); + } +} diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java index 7c6ab4bf2..376193d5a 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/ImplementationKeyValueStorageTest.java @@ -7,19 +7,22 @@ public class ImplementationKeyValueStorageTest extends AbstractSingleFileStorage @Override protected KeyValueStorage buildStringsStorage(String path) { - return new ImplementationKeyValueStorage<>(new SerialiseString(), new SerialiseString(), + return new ImplementationKeyValueStorage<>("String", "Sering", + new SerialiseString(), new SerialiseString(), path) ; } @Override protected KeyValueStorage buildNumbersStorage(String path) { - return new ImplementationKeyValueStorage<>(new SerialiseInteger(), new SerialiseDouble(), + return new ImplementationKeyValueStorage<>("Integer", "Double", + new SerialiseInteger(), new SerialiseDouble(), path) ; } @Override protected KeyValueStorage buildPojoStorage(String path) { - return new ImplementationKeyValueStorage<>(new SerialiseStudentKey(), new SerialiseStudent(), + return new ImplementationKeyValueStorage<>("StudentKey", "Student", + new SerialiseStudentKey(), new SerialiseStudent(), path) ; } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudent.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudent.java index fa855b060..6adba6192 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudent.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudent.java @@ -42,8 +42,4 @@ public Student deserialise(String inString) { return new Student(groupId, name, hometown, birthDate, hasDormitory, averageScore); } - @Override - public String getType() { - return "Student"; - } } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudentKey.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudentKey.java index ec2d9f40e..827451ec3 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudentKey.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task2/SerialiseStudentKey.java @@ -27,9 +27,4 @@ public StudentKey deserialise(String inString) { return new StudentKey(groupId, name); } - @Override - public String getType() { - return "StudentKey"; - } - } diff --git a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java index c6ba9b3c4..e161fe8fc 100644 --- a/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java +++ b/homework-g596-proskurina/src/test/java/ru/mipt/java2016/homework/g596/proskurina/task3/ImplementationKeyValueStoragePerformanceTest.java @@ -1,7 +1,7 @@ package ru.mipt.java2016.homework.g596.proskurina.task3; -import ru.mipt.java2016.homework.base.task2.KeyValueStorage; import ru.mipt.java2016.homework.g596.proskurina.task2.*; +import ru.mipt.java2016.homework.base.task2.KeyValueStorage; import ru.mipt.java2016.homework.tests.task3.KeyValueStoragePerformanceTest; /**