Parsing of API docs completed

apiIndexingTesting.py

@@ -0,0 +1,38 @@
+import os
+
+import log
+import parser
+import models
+import indexer
+from starter import init, end
+import pickler
+import shared
+
+if __name__ == "__main__":
+    init()
+
+    index_dict = dict()
+    api_list = os.listdir('./dataset/API/parsed/')
+
+    ix = 0
+    for aj in api_list:
+        ix += 1
+        if ix % 500 == 0:
+            log.debug(f"Completed {ix}")
+            input()
+
+        aj = os.path.join('./dataset/API/parsed', aj)
+        apiJson = None
+        with open(aj) as f:
+            apiJson = f.read()
+
+        docObject = models.APIDoc.from_json(apiJson)
+
+        index = indexer.index_api_doc(docObject)
+
+        methods_index = index.methods_index
+        index_dict[docObject.name.snippet] = index
+
+    pickler.write_api_index(index_dict)
+    indexer.calculate_all_idf()
+    end()

constants.py

@@ -1,8 +1,25 @@
-
-question_replaceable_special_characters = {',', "'", '"', ';', '?', ':', '-', '(', ')', '[', ']', '{', '}'}
+question_replaceable_special_characters = {',', '//', "'", '"', ';', '?', ':', '-', '(', ')', '[', ']', '{', '}', '$',
+                                           '#',
+                                           '=',
+                                           '<', '>'}
+code_replaceable_special_characters = {',', '//', "'", '"', ';', '?', ':', '-', '(', ')', '[', ']', '{', '}', '$', '#',
+                                       '=',
+                                       '<', '>'}
 special_characters = ['*', '$']
 punctuations = set()
+pickle_data_dir = "bin/data"
 pickled_questions_dir = "bin/data/questions"
+pickled_api_dir = "bin/data/docs"
 pickle_files_extension = ".pickle"
 questions_per_segment = 100
-debug_print_len = 25
+apis_per_segment = 300
+debug_print_len = 25
+question_title_weight = 2
+log_base = 10
+
+max_parallel_threads = 100
+# postgres params
+postgres_host = "127.0.0.1"
+postgres_db = "ir"
+postgres_user = "ir_admin"
+postgres_password = "minda"

crawler.py

@@ -26,3 +26,5 @@ def get_page_soup(url, writeto=None, title=None):
 
 
 
+if __name__ == "__main__":
+    get_page("https://stackoverflow.com/questions/15235400/java-url-param-replace-20-with-space", writeto="./dataset/questions/raw")

dataset/API/parsed/AEADBadTagException.json

@@ -0,0 +1 @@
+{"name": "Class AEADBadTagException", "module": "java.base", "package": "javax.crypto", "text": "This exception is thrown when a Cipher operating in\n an AEAD mode (such as GCM/CCM) is unable to verify the supplied\n authentication tag.", "codes": ["public class AEADBadTagException\nextends BadPaddingException"], "fields": [], "methods": []}

dataset/API/parsed/ATR.json

@@ -0,0 +1 @@
+{"name": "Class ATR", "module": "java.smartcardio", "package": "javax.smartcardio", "text": "A Smart Card's answer-to-reset bytes. A Card's ATR object can be obtained\n by calling Card.getATR().\n This class does not attempt to verify that the ATR encodes a semantically\n valid structure.\n\n Instances of this class are immutable. Where data is passed in or out\n via byte arrays, defensive cloning is performed.", "codes": ["public final class ATR\nextends Object\nimplements Serializable"], "fields": [], "methods": [{"method_name": "getBytes", "method_sig": "public byte[] getBytes()", "description": "Returns a copy of the bytes in this ATR."}, {"method_name": "getHistoricalBytes", "method_sig": "public byte[] getHistoricalBytes()", "description": "Returns a copy of the historical bytes in this ATR.\n If this ATR does not contain historical bytes, an array of length\n zero is returned."}, {"method_name": "toString", "method_sig": "public String toString()", "description": "Returns a string representation of this ATR."}, {"method_name": "equals", "method_sig": "public boolean equals (Object obj)", "description": "Compares the specified object with this ATR for equality.\n Returns true if the given object is also an ATR and its bytes are\n identical to the bytes in this ATR."}, {"method_name": "hashCode", "method_sig": "public int hashCode()", "description": "Returns the hash code value for this ATR."}]}

dataset/API/parsed/AWTError.json

@@ -0,0 +1 @@
+{"name": "Class AWTError", "module": "java.desktop", "package": "java.awt", "text": "Thrown when a serious Abstract Window Toolkit error has occurred.", "codes": ["public class AWTError\nextends Error"], "fields": [], "methods": []}

dataset/API/parsed/AWTEvent.json

@@ -0,0 +1 @@
+{"name": "Class AWTEvent", "module": "java.desktop", "package": "java.awt", "text": "The root event class for all AWT events.\n This class and its subclasses supersede the original\n java.awt.Event class.\n Subclasses of this root AWTEvent class defined outside of the\n java.awt.event package should define event ID values greater than\n the value defined by RESERVED_ID_MAX.\n \n The event masks defined in this class are needed by Component subclasses\n which are using Component.enableEvents() to select for event types not\n selected by registered listeners. If a listener is registered on a\n component, the appropriate event mask is already set internally by the\n component.\n \n The masks are also used to specify to which types of events an\n AWTEventListener should listen. The masks are bitwise-ORed together\n and passed to Toolkit.addAWTEventListener.", "codes": ["public abstract class AWTEvent\nextends EventObject"], "fields": [{"field_name": "id", "field_sig": "protected\u00a0int id", "description": "The event's id."}, {"field_name": "consumed", "field_sig": "protected\u00a0boolean consumed", "description": "Controls whether or not the event is sent back down to the peer once the\n source has processed it - false means it's sent to the peer; true means\n it's not. Semantic events always have a 'true' value since they were\n generated by the peer in response to a low-level event."}, {"field_name": "COMPONENT_EVENT_MASK", "field_sig": "public static final\u00a0long COMPONENT_EVENT_MASK", "description": "The event mask for selecting component events."}, {"field_name": "CONTAINER_EVENT_MASK", "field_sig": "public static final\u00a0long CONTAINER_EVENT_MASK", "description": "The event mask for selecting container events."}, {"field_name": "FOCUS_EVENT_MASK", "field_sig": "public static final\u00a0long FOCUS_EVENT_MASK", "description": "The event mask for selecting focus events."}, {"field_name": "KEY_EVENT_MASK", "field_sig": "public static final\u00a0long KEY_EVENT_MASK", "description": "The event mask for selecting key events."}, {"field_name": "MOUSE_EVENT_MASK", "field_sig": "public static final\u00a0long MOUSE_EVENT_MASK", "description": "The event mask for selecting mouse events."}, {"field_name": "MOUSE_MOTION_EVENT_MASK", "field_sig": "public static final\u00a0long MOUSE_MOTION_EVENT_MASK", "description": "The event mask for selecting mouse motion events."}, {"field_name": "WINDOW_EVENT_MASK", "field_sig": "public static final\u00a0long WINDOW_EVENT_MASK", "description": "The event mask for selecting window events."}, {"field_name": "ACTION_EVENT_MASK", "field_sig": "public static final\u00a0long ACTION_EVENT_MASK", "description": "The event mask for selecting action events."}, {"field_name": "ADJUSTMENT_EVENT_MASK", "field_sig": "public static final\u00a0long ADJUSTMENT_EVENT_MASK", "description": "The event mask for selecting adjustment events."}, {"field_name": "ITEM_EVENT_MASK", "field_sig": "public static final\u00a0long ITEM_EVENT_MASK", "description": "The event mask for selecting item events."}, {"field_name": "TEXT_EVENT_MASK", "field_sig": "public static final\u00a0long TEXT_EVENT_MASK", "description": "The event mask for selecting text events."}, {"field_name": "INPUT_METHOD_EVENT_MASK", "field_sig": "public static final\u00a0long INPUT_METHOD_EVENT_MASK", "description": "The event mask for selecting input method events."}, {"field_name": "PAINT_EVENT_MASK", "field_sig": "public static final\u00a0long PAINT_EVENT_MASK", "description": "The event mask for selecting paint events."}, {"field_name": "INVOCATION_EVENT_MASK", "field_sig": "public static final\u00a0long INVOCATION_EVENT_MASK", "description": "The event mask for selecting invocation events."}, {"field_name": "HIERARCHY_EVENT_MASK", "field_sig": "public static final\u00a0long HIERARCHY_EVENT_MASK", "description": "The event mask for selecting hierarchy events."}, {"field_name": "HIERARCHY_BOUNDS_EVENT_MASK", "field_sig": "public static final\u00a0long HIERARCHY_BOUNDS_EVENT_MASK", "description": "The event mask for selecting hierarchy bounds events."}, {"field_name": "MOUSE_WHEEL_EVENT_MASK", "field_sig": "public static final\u00a0long MOUSE_WHEEL_EVENT_MASK", "description": "The event mask for selecting mouse wheel events."}, {"field_name": "WINDOW_STATE_EVENT_MASK", "field_sig": "public static final\u00a0long WINDOW_STATE_EVENT_MASK", "description": "The event mask for selecting window state events."}, {"field_name": "WINDOW_FOCUS_EVENT_MASK", "field_sig": "public static final\u00a0long WINDOW_FOCUS_EVENT_MASK", "description": "The event mask for selecting window focus events."}, {"field_name": "RESERVED_ID_MAX", "field_sig": "public static final\u00a0int RESERVED_ID_MAX", "description": "The maximum value for reserved AWT event IDs. Programs defining\n their own event IDs should use IDs greater than this value."}], "methods": [{"method_name": "setSource", "method_sig": "public void setSource (Object newSource)", "description": "Retargets an event to a new source. This method is typically used to\n retarget an event to a lightweight child Component of the original\n heavyweight source.\n \n This method is intended to be used only by event targeting subsystems,\n such as client-defined KeyboardFocusManagers. It is not for general\n client use."}, {"method_name": "getID", "method_sig": "public int getID()", "description": "Returns the event type."}, {"method_name": "toString", "method_sig": "public String toString()", "description": "Returns a String representation of this object."}, {"method_name": "paramString", "method_sig": "public String paramString()", "description": "Returns a string representing the state of this Event.\n This method is intended to be used only for debugging purposes, and the\n content and format of the returned string may vary between\n implementations. The returned string may be empty but may not be\n null."}, {"method_name": "consume", "method_sig": "protected void consume()", "description": "Consumes this event, if this event can be consumed. Only low-level,\n system events can be consumed"}, {"method_name": "isConsumed", "method_sig": "protected boolean isConsumed()", "description": "Returns whether this event has been consumed."}]}

dataset/API/parsed/AWTEventListener.json

@@ -0,0 +1 @@
+{"name": "Interface AWTEventListener", "module": "java.desktop", "package": "java.awt.event", "text": "The listener interface for receiving notification of events\n dispatched to objects that are instances of Component or\n MenuComponent or their subclasses.  Unlike the other EventListeners\n in this package, AWTEventListeners passively observe events\n being dispatched in the AWT, system-wide.  Most applications\n should never use this class; applications which might use\n AWTEventListeners include event recorders for automated testing,\n and facilities such as the Java Accessibility package.\n \n The class that is interested in monitoring AWT events\n implements this interface, and the object created with that\n class is registered with the Toolkit, using the Toolkit's\n addAWTEventListener method.  When an event is\n dispatched anywhere in the AWT, that object's\n eventDispatched method is invoked.", "codes": ["public interface AWTEventListener\nextends EventListener"], "fields": [], "methods": [{"method_name": "eventDispatched", "method_sig": "void eventDispatched (AWTEvent event)", "description": "Invoked when an event is dispatched in the AWT."}]}

dataset/API/parsed/AWTEventListenerProxy.json

@@ -0,0 +1 @@
+{"name": "Class AWTEventListenerProxy", "module": "java.desktop", "package": "java.awt.event", "text": "A class which extends the EventListenerProxy\n specifically for adding an AWTEventListener\n for a specific event mask.\n Instances of this class can be added as AWTEventListeners\n to a Toolkit object.\n \n The getAWTEventListeners method of Toolkit\n can return a mixture of AWTEventListener\n and AWTEventListenerProxy objects.", "codes": ["public class AWTEventListenerProxy\nextends EventListenerProxy<AWTEventListener>\nimplements AWTEventListener"], "fields": [], "methods": [{"method_name": "eventDispatched", "method_sig": "public void eventDispatched (AWTEvent event)", "description": "Forwards the AWT event to the listener delegate."}, {"method_name": "getEventMask", "method_sig": "public long getEventMask()", "description": "Returns the event mask associated with the listener."}]}