Benchmark

12-03-2009, 04:33 PM

Hey,
This time I want to be sure before submitting anything.
The code is not entirely finished.
But what do you think about this:

You can read the documentation in the hidden part.

JASS:

library Benchmark initializer Init
    //   |===========================================|
    //   |============= S E T T I N G S =============|
    //   |===========================================|
    //             === DISABLE_ERROR ===
    // This messages will be displayed, if Emmeasure was disabled, because it hasn't been used
    // properly.
    //            === ABORT_ERROR ===
    // This message will be displayed, if a Benchmark was aborted, because it hasn't been used
    // properly.
    //            === COLOR_MASSIVELY ===
    // Changes the Color Algorithm, so if this is true, the Colors are more glaring and you can
    // see the speed differnces clearly. [Read PF 1]
    //             === UNIQUE_ERRORS ===
    // If this is true, the system will not spam error messages, it will display each unique message
    // once. Example: It will not display 'I am nobus m' twice, even if the error appears 100 times or more.
    // [ Read PF 2 ]
    //             === QUALITY ===
    // This shouldn't be changed in order to make tests more uniform. I think I'll add a protection later, so
    // that people can't cheat.
    // What it does? It Displays how good the test has been at the End.
    // If you tested 30 000 times for example, it would display 'safe'.
    // And if you test 500 times, it would display 'unacceptable'.

    //             === [PF 1] ===
    // Performance Cost 1: The system itself gets a tiny little bit slower, so it might lag more, but this doesn't affect
    // the speed of other systems.
    //             === [PF 2] ===
    // Performance Cost 2: The system itself gets a little bit slower, so it might lag more, but this doesn't affect
    // the speed of other systems.
    // ============= Thanks to =============
    // - Buster4 for Per2Clr and Int2Hex
    // - XieLong for a bit help with the colorcode algorithms.
    // - Vexorian for JassNewgNen
    // - Skater for Dec2Hex
    globals
        private constant integer NOTICEABLE_FPS    =  100 // Eye can see 32 pcitures per second.
        private constant string DISABLE_ERROR      =  "Emmeasure was Disabled."
        private constant string ABORT_ERROR        =  "Aborting Benchmark!"
        private constant boolean VISUALIZE         =  false // Requires TestHelper used.
        private constant boolean COLOR_MASSIVELY   =  true
        private constant boolean UNIQUE_ERRORS     =  false
        private constant boolean TEST_FUNC_SPEED   =  true // This measures the speed of single functions. Costs a lot of speed! 
                                                            // WARNING: This costs a lot performance, because funcs are registered 
                                                            // by the system and use 2D Indexes.
        private constant real TYPO_MIN_MATCH       =  40. // Do not change!
        private constant integer QUALITY           =  15000 // Do not change!
        private constant integer VISION_VALUE      =  100000 // Do not change!
    endglobals
    
    globals
        private real array Time // How much time did a sys take
        private integer array Runs // How many times was it called?
        private real t0 // temp value
        private real t1 // temp value
        private integer Clock // StopWatch
        private integer System // temp Value
        private boolean Running = false // temp Value
        private integer sysc = 0 // How many systems did you declare?
        private string array sysn // Name
        private integer array sysm 
        private string array StableWord // Like 'Excellent'
        private real array FuncSpeed // For functions: How fast they've been
        private real array Dividend // Rate/Frequency
        private integer array FuncExe // How often has a function be called?
        private string array Graph // Graph created at comparison
        private boolean DoTest = true // false = Abort Benchmark.
        private string TempFunc // Which function is just tested?
        private trigger trig = CreateTrigger()
        private trigger t = CreateTrigger()
        private boolean RegisterSystems = true
        private boolean Test_Closed = true
        private integer Won = 0 // Which System (Index of string array) was the fastest?
        private integer instance = 0 // Which instance is tested at the testhepler?
        private multiboard table
        private integer ttt = 0
    endglobals    
    
// NOT MY CODE
function Int2Hex takes integer int returns string
    local string charMap = "0123456789ABCDEF"
    local string hex = ""
    local integer index = 0
    if ((int < 0) or (int > 0x7FFFFFFF)) then
        return "|cffff0000Invalid argument in function Int2Hex()!|r"
    endif
    loop
        set index = ModuloInteger(int, 0x10) + 1
        set int = int / 0x10
        set hex = SubStringBJ(charMap, index, index) + hex
        exitwhen (int == 0)
    endloop
    return hex
endfunction

function Per2Clr takes string text, integer r, integer g, integer b, integer alpha returns string
    local string array str 
    if ((r >= 0) and (r <= 100) and (g >= 0) and (g <= 100) and (b >= 0) and (b <= 100) and (alpha >= 0) and (alpha <= 100)) then
        set r = R2I(0xFF / 100.000 * r)
        set g = R2I(0xFF / 100.000 * g)
        set b = R2I(0xFF / 100.000 * b)
        set alpha = R2I(0xFF / 100.000 * alpha)
        if (alpha <= 0xF) then
            set str[0] = "0" + Int2Hex(alpha)
        else
            set str[0] = Int2Hex(alpha)
        endif
        if (r <= 0xF) then
            set str[1] = "0" + Int2Hex(r)
        else
            set str[1] = Int2Hex(r)
        endif
        if (g <= 0xF) then
            set str[2] = "0" + Int2Hex(g)
        else
            set str[2] = Int2Hex(g)
        endif
        if (b <= 0xF) then
            set str[3] = "0" + Int2Hex(b)
        else
            set str[3] = Int2Hex(b)
        endif 
        return "|c" + str[0] + str[1] + str[2] + str[3] + text
    else
        return "|cffff0000Invalid arguments in function Per2Clr()!|r"
    endif        
endfunction

function Dec2Hex takes integer i returns string
    local string charMap="0123456789ABCDEF"
    local string hex
    if i>255 then
        return "|cffff0000This function is NOT designed to convert values above 255!|r"
    elseif i<0 then
        return "|cffff0000 The value is to small!|r"
    endif
    if i<16 then
        return "0"+SubString(charMap, i, i+1)
    else
        set hex=SubString(charMap, i/16, i/16+1)
        set i=ModuloInteger(i, 16)
        set hex=hex+SubString(charMap, i, i+1)
        return hex
    endif
endfunction
    
// ====== MY CODE =======
private function CreatePercentageBar takes real Percent returns string
    local real Red
    local real Green
    local string ret
    local integer i = 0
    
    if COLOR_MASSIVELY then
        if Percent <= 67 then
            set Red = 100
            set Green = 100 * (Percent / 67)
        else
            //set Red = 300 - (100/33.3333*Percent) // Ist bereits ausgekürzt, eig 100 - (100 * (Procent - 66) / 33)
            //set Red = 100-Percent // 100 bis 0
            set Red = 3*(100-Percent)
            set Green = 100
        endif
    else
        set Red = 100-Percent
        set Green = Percent
    endif
    set ret = "["+ Per2Clr(" ",R2I(Red),R2I(Green),0,0)
    
    loop
        set i = i + 1
        exitwhen i == 100
        if i-1 == R2I(Percent) then
        
            if ModuloInteger(i,2) == 1 then
                set ret = ret + "|r"
            else
                set ret = ret + "||r"
            endif
            
            set ret = ret + "|cff808080"
        endif
        set ret = ret + "|"
    endloop
    return ret+" |r]"
    
endfunction

    globals
        private integer ErrorCount = 0
        private string array Errors
        private integer TypoCount = 0
        private string array Typos
        private integer array TypoResult
    endglobals
    
function BenchError takes string s returns nothing
    //: We don't want to spam error messages, so each unique error is only displayed once.
    local integer i = 0
    if UNIQUE_ERRORS then
        loop
            set i = i + 1
            exitwhen i > ErrorCount
            if Errors[i] == "|cffff0000"+s+"|r" then
                return
            endif
        endloop
    endif
    //: Error is unique: Register.
    set ErrorCount = ErrorCount + 1
    set Errors[ErrorCount] = "|cffff0000"+s+"|r"
    call BJDebugMsg(Errors[ErrorCount])
endfunction

    globals
        private integer EPT = 100
        private integer Threads = 100
        private integer CURRENT_EPT = 0
        private integer CURRRENT_Threads = 0
        private boolean Finished = false
        private boolean New = false
    endglobals
    
struct TestHelper
    static method SetTest takes integer ept, integer threads returns nothing
        if Test_Closed then
            set instance = 0
            set EPT = ept
            set Threads = threads
            set Finished = false
            set Test_Closed = false
        else
            call BenchError("Started closed test during another one (TestHelper struct).")
            call BenchError(ABORT_ERROR)
            call Test.Disable()
        endif
    endmethod
    
    static method NextInstance takes nothing returns nothing
        set CURRENT_EPT = CURRENT_EPT + 1
        if CURRENT_EPT > EPT then
            set CURRENT_EPT = 0
            set CURRRENT_Threads = CURRRENT_Threads + 1
            set New = true
            if CURRRENT_Threads > Threads then
                set Finished = true
                set Test_Closed = true
            endif
        endif
        if VISUALIZE then
            set instance = instance + 1
            call ClearTextMessages()
            call BJDebugMsg("Completed: "+R2S(instance/(EPT*Threads))+"%")
        endif
    endmethod
    
    static method IsThreadNew takes nothing returns boolean
        local boolean b = New
        if b then
            set New = false
        endif
        return b
    endmethod
    
    static method IsFinished takes nothing returns boolean
        return Finished
    endmethod
endstruct

private function I2D takes integer unlimited, integer sized, integer size returns integer
    return sized+(unlimited*size)
endfunction

    globals
        private string currentFunc
        private string array Funcs
        private integer array FuncNum
    endglobals
    
    
struct Test
    static method Declare takes string name returns nothing
        //if RegisterSystems then
            set sysc = sysc + 1
            set sysn[sysc] = name
            set sysm[sysc] = 0
        //else
        //    call BenchError("TEST: You may only declare systems in intitializers.")
        //endif
    endmethod
    
    // ====== ========= ====== Gets unique Indexes for Systems and functions.
    // ====== CONVERTER ======
    // ====== ========= ======
    static method Convert takes string name returns integer
        local integer i = 0
        loop
            set i = i + 1
            exitwhen i > sysc
            if name == sysn[i] then
                return i
            endif
        endloop
        call BenchError("Didn't find System: "+name )
        call BenchError(ABORT_ERROR)
        call Test.Disable()
        return 0 // Don't return -1, it would crash wc3.
    endmethod
    
    static method ConvertFunc takes integer con, string funcname returns integer
        //local integer con = Test.Convert(name)
        // Uses 2D Indexes   :    NumberOfFunction, NumberOfSystem
        local integer i = 0
        
        loop
            set i = i + 1
            exitwhen i > FuncNum[con]
            if funcname == Funcs[I2D(i,con,sysc)] then
                //call BJDebugMsg("Found at "+I2S(I2D(i,con,sysc))+ " | "+I2S(i))
                return I2D(i,con,sysc)
            endif
        endloop
        //call BJDebugMsg("Registered at "+I2S(I2D(FuncNum[con],con,sysc)))
         // We don't use the var for loops anymore.
        set FuncNum[con] = FuncNum[con] + 1
        set i = I2D(FuncNum[con],con,sysc)
        set Funcs[i] = funcname
        set Dividend[i] = 1
        return I2D(FuncNum[con],con,sysc)
    endmethod
    
    // ====== ========= =======
    // ===== ENDCONVERTER =====
    // ====== ========= =======
    
    static method DisplayErrors takes nothing returns nothing
        local integer i = 0
        loop
            set i = i + 1
            exitwhen i > ErrorCount
            call BJDebugMsg("Error Nr."+I2S(i)+": "+Errors[i])
        endloop
    endmethod
    
    static method SetFuncDividend takes string Name, string FuncName, real Divisor returns nothing
        set Dividend[Test.ConvertFunc(Test.Convert(Name),FuncName)] = Divisor
    endmethod
    
    static method Disable takes nothing returns nothing
        set DoTest = false
    endmethod
    
    static method Start takes string Name, string FuncName returns integer
        if DoTest then
            if Running == false then
                set TempFunc = FuncName
                set Running = true
                set System = Test.Convert(Name)
                set Runs[System] = Runs[System] + 1
                set t0 = StopWatchMark(Clock)
                return 0
            else
                call BJDebugMsg("TEST: System "+sysn[System]+", function "+TempFunc+" is causing an open thread when calling "+FuncName+".")
                call BenchError(ABORT_ERROR)
                call Test.Disable()
            endif
        endif
        return 0
    endmethod
    
    static method End takes nothing returns integer
       set t1 = StopWatchMark(Clock)
        if DoTest then
            if Running then
                set Time[System] = Time[System] + (t1-t0)
                
                if TEST_FUNC_SPEED then
                    set ttt = Test.ConvertFunc(System,TempFunc)
                    //call BJDebugMsg("Int for "+TempFunc+" - "+I2S(System)+": "+I2S(funcInt)) 
                    set FuncExe[ttt] = FuncExe[ttt] + 1
                    set FuncSpeed[ttt] = FuncSpeed[ttt] + (t1-t0)
                endif
                
                set System = 0
                set Running = false
            endif
        endif
        return 0
    endmethod
    
    static method Display takes string Name returns nothing
        local integer sys = Test.Convert(Name)
        if DoTest then
            call BJDebugMsg(Name+ " took "+R2S(Time[sys])+ " seconds." )
        else
            call BenchError(DISABLE_ERROR)
        endif
    endmethod
    
    static method DisplayFunc takes string Name, string FuncName returns nothing
        local integer sys = Test.ConvertFunc(Test.Convert(Name),FuncName)
        if DoTest then
            if TEST_FUNC_SPEED == false then
                call BenchError("TEST: Started DisplayFunc without having TEST_FUNC_SPEED enabled.")
                return
            endif
            call BJDebugMsg("Speed of "+Per2Clr(Name,100,80,20,0)+"|r function "+Per2Clr(FuncName,0,100,0,0)+"|r: "+R2S(FuncSpeed[sys]*VISION_VALUE/FuncExe[sys]/Dividend[sys])+ " EPS.")
        else
            call BenchError(DISABLE_ERROR)
        endif
    endmethod
    
    static method CompareFuncs takes integer FuncIndex returns nothing
        local integer i = 0
        local integer funcInd = 0
        if DoTest == false then
            call BenchError(DISABLE_ERROR)
        return
        endif
        
        if TEST_FUNC_SPEED then
            loop
                set i = i + 1
                exitwhen i > sysc
                set funcInd = I2D(FuncIndex,i,sysc)
                call BJDebugMsg("Speed of "+Per2Clr(sysn[i],100,80,20,0)+"|r, function "+Per2Clr(Funcs[funcInd],0,100,0,0)+"|r: "+R2S(FuncSpeed[funcInd]*VISION_VALUE/FuncExe[funcInd]/Dividend[funcInd])+ " EPS.")
            endloop
        endif
    endmethod
    
    static method Compare takes nothing returns nothing
        local integer i = 0
        local real highest = Time[1]
        local real perc
        local real Exec = 0.
        local integer Temp
        if DoTest == false then
            call BenchError(DISABLE_ERROR)
            return
        endif
        set Won = 1
        loop
            set i = i + 1
            exitwhen i > sysc
            if Time[i] < highest then
                set highest = Time[i]
                //call BJDebugMsg("Won; "+I2S(i))
                set Won = i
            endif
        endloop
        
        set i = 0
        call BJDebugMsg("================================")
        loop
            set i = i + 1
            exitwhen i > sysc
            set Exec = Exec + Runs[i]
            set perc = highest/Time[i]*100.
            call BJDebugMsg(sysn[i]+": "+R2S(perc)+"%")
            set Graph[i] = CreatePercentageBar(perc)
            call BJDebugMsg(Graph[i])
        endloop
        set Exec = Exec / sysc
        set Temp = R2I(Exec/QUALITY)
        if Exec >= 500*500 then
            set Temp = 8
        else
            if Temp > 7 then
                set Temp = 7
            endif
        endif
        call BJDebugMsg("Testing Quality: "+StableWord[Temp])
        debug call BJDebugMsg(Per2Clr("Test started in Debug Mode!",80,30,0,0)+"|r")
        call BJDebugMsg("Emmeasure by Mr.Malte")
    endmethod
    
    // This part is kinda ugly, I used many BJs - but I don't care.
    // It hasn't to be fast, it's only executed once and the BJs save some work.
    static method createTable takes nothing returns nothing
        local integer i = 0
         local integer i2 = 0
         local integer funcInd = 0
         local integer cc = sysc+1+2
        local real arVa1
        local real temp
        local real array Rate
         local real arVa2
         if Won == 0 then
            call Test.Compare()
            call ClearTextMessages()
         endif
            set table = CreateMultiboardBJ( 4, 20, "Test Information" )
            debug call MultiboardSetTitleText(table,"Test Information (Debug Mode)")
            call MultiboardSetItemStyleBJ( table, 0, 0, true, false )
        //call MultiboardSetTitleText(table,"Emmeasure Test")
        //call MultiboardSetRowCount(table,1+sysc)
        //call MultiboardSetColumnCount(table,4)
        call MultiboardSetItemWidthBJ( table, 0, 0, 7.00 )
        call MultiboardSetItemWidthBJ( table, 2, 0, 12.00 )
        call MultiboardSetItemWidthBJ( table, 1, 0, 13.50 )
        
        call MultiboardSetItemColorBJ(table,0,1,30,70,90,0)
        call MultiboardSetItemValueBJ(table,1,1,"System")
        call MultiboardSetItemValueBJ(table,2,1,"Speed")
        call MultiboardSetItemValueBJ(table,3,1,"EPF*")
        call MultiboardSetItemValueBJ(table,4,1,"CPF**")
        set cc = cc + 1
        call MultiboardSetItemColorBJ(table,0,cc,30,70,90,0)
        call MultiboardSetItemValueBJ(table,1,cc,"Function")
        call MultiboardSetItemValueBJ(table,2,cc,"Speed")
        call MultiboardSetItemValueBJ(table,3,cc,"Rate")
        // Declare Rate: How often systems are executed.
        set i = 0
        loop
            set i = i + 1
            set i2 = 0
            exitwhen i > sysc
            set Rate[i] = 0
            loop
                set i2 = i2 + 1
                exitwhen i2 > FuncNum[i]
                set funcInd = I2D(i2,i,sysc)
                set Rate[i] = Rate[i] + Dividend[funcInd]
                
            endloop
            //call BJDebugMsg("Rate ["+I2S(i)+"] - "+I2S(Rate[i]))
        endloop
        
        set i2 = 0
        set i = 0
        loop
            set i = i + 1
            set i2 = 0
            exitwhen i > sysc
            call MultiboardSetItemValueBJ(table,1,i+1,sysn[i])
            call MultiboardSetItemValueBJ(table,1,i+1+sysc,sysn[i])
            call MultiboardSetItemValueBJ(table,2,i+1,Graph[i])
            set cc = cc + 1
            call MultiboardSetItemColorBJ(table,1,cc,100,80,20,0)
            call MultiboardSetItemValueBJ(table,1,cc,"   -"+sysn[i])
            set arVa1 = 0.
            set arVa2 = 0.
            // Calculate arithmetric value of speef of the funcs.
            loop
                set i2 = i2 + 1
                exitwhen i2 > FuncNum[i]
                set funcInd = I2D(i2,i,sysc)
                set temp = FuncSpeed[funcInd]/FuncExe[funcInd]/Dividend[funcInd]
                set arVa1 = arVa1 + temp
                set arVa2 = arVa2 + (temp*Dividend[funcInd])
                set cc = cc + 1
                // Add the function values
                call MultiboardSetItemValueBJ(table,1,cc,Funcs[funcInd])
                call MultiboardSetItemValueBJ(table,2,cc,R2S(temp*VISION_VALUE))
                call MultiboardSetItemValueBJ(table,3,cc,R2S(Dividend[funcInd]/Rate[i]*100.)+"%")
            endloop
            
            set arVa1 = Runs[i]/arVa1
            set arVa2 = Runs[i]/arVa2
            // Add the System speed values
            call MultiboardSetItemValueBJ(table,3,i+1,I2S(R2I(arVa1/NOTICEABLE_FPS/VISION_VALUE)))
            call MultiboardSetItemValueBJ(table,4,i+1,I2S(R2I(arVa2/NOTICEABLE_FPS/VISION_VALUE)))
        endloop
        
        call MultiboardSetItemValueBJ(table,2,sysc+2,"*k Executions per Frame")
        call MultiboardSetItemValueBJ(table,2,sysc+3,"** Cycles per Frame")
        call MultiboardSetItemValueBJ(table,1,sysc+2,"")
        call MultiboardSetItemValueBJ(table,1,sysc+3,"")
        call MultiboardSetItemValueBJ(table,1,sysc+4,"System")
        set cc = cc + 2
        call BJDebugMsg(Per2Clr(sysn[Won],80,30,0,0))
        call MultiboardSetItemValueBJ(table,2,cc,"Testwinner: "+Per2Clr(sysn[Won],80,30,0,0)+"|r")
        call MultiboardDisplay(table,true)
    endmethod
endstruct



private function AbortDebug takes nothing returns nothing
    // Debug mode can be detected, but I don't know how to detect Wc3err.
    // Commented this to allow tests in Debug mode.
    //debug call BenchError("Test was started in Debug Mode; Disabling Benchmark!")
    //debug call Test.Disable()
    set RegisterSystems = false
    // Test Quality
    if QUALITY/((3*5*5*2*2*2*2*5)+(9*50*20)) == 1 and QUALITY < 243094 and QUALITY > 5432 then
    else
        call BenchError("Constant 'QUALITY' was changed; Disabling Benchmark")
        call Test.Disable()
    endif
endfunction

private function Init takes nothing returns nothing
    set Clock = StopWatchCreate()
    call TriggerRegisterTimerEventSingle(t,0.001)
    call TriggerAddAction(t,function AbortDebug)
    set StableWord[0] = Per2Clr("unacceptable",100,0,0,0)
    set StableWord[1] = Per2Clr("poor",86,14,0,0)
    set StableWord[2] = Per2Clr("lacking",71,28,0,0)
    set StableWord[3] = Per2Clr("not bad",100,100,0,0)
    set StableWord[4] = Per2Clr("fine",45,70,0,0)
    set StableWord[5] = Per2Clr("safe",30,80,0,0)
    set StableWord[6] = Per2Clr("Great",20,80,0,0)
    set StableWord[7] = Per2Clr("Excellent",0,100,0,0)
    set StableWord[8] = Per2Clr("Perfect",0,100,0,0)
endfunction
endlibrary

so far?
It is a system used to test the speed of functions using StopWatch. A collection of fancy wrappers.

I planned this Introduction:

Hidden information:

Example of a perfect Test with Benchmark
TimerUtils vs. Table

JASS:

library TimerUtilsTest initializer SetUp requires TestWrappers, TimerUtils, Table

    globals
        private HandleTable Data
    endglobals
    
private function test takes nothing returns nothing
    local timer t
    local integer i
    call TestHelper.NextInstance()

        if TestHelper.IsFinished() then
            call PauseTimer(GetExpiredTimer())
            call DestroyTimer(GetExpiredTimer())
            call Test.Compare()
            call Test.createTable()
            return
        endif
    
        // TimerUtils
        call Test.Start("TimerUtils","NewTimer")
            set t = NewTimer()
        call Test.End()

        call Test.Start("TimerUtils","SetTimerData")
            call SetTimerData(t,1)
        call Test.End()
            
        call Test.Start("TimerUtils","GetTimerData")
            set i = GetTimerData(t)
        call Test.End()
            
        call Test.Start("TimerUtils","ReleaseTimer")
            call ReleaseTimer(t)
        call Test.End()
            
        set t = null
        
        call Test.Start("Natives","CreateTimer")
            set t = CreateTimer()
        call Test.End()

        call Test.Start("Natives","Assign")
            set Data[t] = 1
        call Test.End()
            
        call Test.Start("Natives","Get")
            set i = Data[t]
        call Test.End()
            
        call Test.Start("Natives","Flush")
            call Data.flush(t)
        call Test.End()
            
        call Test.Start("Natives","DestroyTimer")
            call DestroyTimer(t)
        call Test.End()

endfunction

    function TestTimerUtils takes nothing returns nothing
        call TestHelper.SetTest(500,500)
        call TimerStart(CreateTimer(),0.,true,function test)
    endfunction
    
    private function SetUp takes nothing returns nothing
        set Data = HandleTable.create()
        call Test.Declare("TimerUtils")
        call Test.Declare("Natives")
    endfunction

endlibrary

Result

Attached Files

Benchmark.w3x (67.4 KB)

12-03-2009, 06:19 PM	#2
Anitarf	I've read through the documentation and I still have no idea how the heck do you do your benchmarks. Furthermore, including non-functional fluff like MakeGradientText doesn't make this a more useful resource.

12-03-2009, 06:43 PM	#3
Rising_Dusk	Why would anyone use this over, say, the Stopwatch natives...?

12-03-2009, 07:36 PM

Mr.Malte

Quote:

I've read through the documentation and I still have no idea how the heck do you do your benchmarks.

How I do my benchmarks or how you (Antiarf) do your benchmarks?

Quote:

Furthermore, including non-functional fluff like MakeGradientText doesn't make this a more useful resource.

True. Will remove it.

Quote:

Why would anyone use this over, say, the Stopwatch natives...?

These are just wrappers for the Stopwatch natives that make testing easier.
Well, I did not make a list of advantages.
If you want to find out all the advantages you have to read thrugh all the hidden stuff, I'm sorry.
A raw summarize:

detects open threads
makes nice graphical compares
helps you making accurate tests
makes the work with the stopwatch natives much easier.

You can take a look at miy TimerUtils example.

Let me quote grim:

Quote:

Originally Posted by grim001

Err I am pretty sure this system does use the stopwatch natives, it just adds a bunch of fancy wrappers to make testing systems easier.

edit:

The biggest problem of this system I see right now are these uberfeatures:
- Displaying errors uniquely (That's where Optibug came from)
- Fixing typos you did when writing system names

But I'm not sure whether they can stay or not.
They cause code bloat, but they are not negative and since this uses StopWatch and will never run in a real map, I think actually it's not important.

12-03-2009, 07:53 PM

akolyt0r

i read your documentation, but have no clue what following functions do:

JASS:

// Test.RegisterInstance(sysname)      : Registers, that a new instance of the test has started.
// Test.RemoveInstance(sysname)        : Removes a registered Instance.
// Test.RunThisCode(sysname)           : Returns, if the system has exactly one registered Instance.
// Test.RunCode()                      : Displays if no System Instances are running.

documentation is still quite lacking...
and that "detect typos" thing is totally unneccessary

12-03-2009, 08:12 PM

Mr.Malte

That's true.
Will remove it, too.
The documentation of the system iteself is lacking, that's true, too.
But I will make a proper documentation for it later. The actual documentation is hidden in the first post.

Oh, I removed the part you quoted anyways.
It was useless.

@ Antiarf: No I understand.
You read the documentation of the system itself.
I will rework it.
If you want to understand it, you should read the hidden part.

12-03-2009, 08:49 PM

Rising_Dusk

Quote:

Originally Posted by Mr.Malte

How I do my benchmarks or how you (Antiarf) do your benchmarks?

Well, seeing as how he was talking to you and used the word 'you', it would suggest he is wondering how you do your benchmarks.

Quote:

Originally Posted by Mr.Malte

These are just wrappers for the Stopwatch natives that make testing easier.

It's strange, because in the few cases I've ever actually benchmarked anything, they were already stupid-easy to use. I can't imagine needing any wrapper syntax for it at all. I mean, basically your library does nothing useful, it just creates a multiboard to display some stats from benchmarking.

12-05-2009, 01:37 PM

Mr.Malte

Quote:

your library does nothing useful, it just creates a multiboard to display some stats from benchmarking.

It compares all the systems. It puts out nice graphics where you can just take a screen of to proof your benchmarks. Graphics and such things like multiboards are always better to have a nice comparison.

Of course this doesn't seem useful for such an easy benchmark like TimerUtils.
But it becomes useful in more complicated systems like AutoIndex.

12-05-2009, 05:22 PM

Rising_Dusk

AutoIndex is just as easy a benchmark, really.

Quote:

Originally Posted by Mr.Malte

It puts out nice graphics where you can just take a screen of to proof your benchmarks

BJDebugMsg puts out sufficient graphics for that purpose.

If that's the best defense you can muster for your script, this really isn't useful. :p

12-05-2009, 06:20 PM

#10

Mr.Malte

Well, the library gets a little longer without Benchmark and doesn't display nice graphics:

JASS:

scope TimerUtilsTest initializer SetUp

    globals
        HandleTable Data
        integer Clock
        real t0
        real t1
        real array time
        integer count = 0
    endglobals
    
private function test takes nothing returns nothing
    local timer t
    local integer i

    set count = count + 1
        if count > 250000
            call PauseTimer(GetExpiredTimer())
            call DestroyTimer(GetExpiredTimer())
            // Getting more complex with more systems
            if time[0]+time[1]+time[2]+time[3] > time[4]+time[5]+time[6]+time[7]+time[8] then
                call BJDebugMsg("TimerUtils: 100%")
                call BJDebugMsg("Natives: "+R2S((time[4]+time[5]+time[6]+time[7]+time[8])/(time[0]+time[1]+time[2]+time[3]))+"%")
            else
                call BJDebugMsg("Natives: 100%")
                call BJDebugMsg("TimerUtils: "+R2S((time[0]+time[1]+time[2]+time[3])/(time[4]+time[5]+time[6]+time[7]+time[8]))+"%")
            endif
            call BJDebugMsg("Time of TimerUtils: "+R2S(time[0]+time[1]+time[2]+time[3]))
            call BJDebugMsg("NewTimer: "+R2S(time[0]*100000))
            call BJDebugMsg("SetTimerData: "+R2S(time[1]*100000))
            call BJDebugMsg("GetTimerData: "+R2S(time[2]*100000))
            call BJDebugMsg("ReleaseTimer: "+R2S(time[3]*100000))
            call BJDebugMsg("Time of Natives: "+R2S(time[4]+time[5]+time[6]+time[7]+time[8]))
            call BJDebugMsg("CreateTimer: "+R2S(time[4]*100000))
            call BJDebugMsg("Assign: "+R2S(time[5]*100000))
            call BJDebugMsg("Get: "+R2S(time[6]*100000))
            call BJDebugMsg("Flush: "+R2S(time[7]*100000))
            call BJDebugMsg("DestroyTimer: "+R2S(time[8]*100000))
            return
        endif
    
        // TimerUtils
        set t0 = StopWatchMark(Clock)
        set t = NewTimer()
        set t1 = StopWatchMark(Clock)
        set time[0] = t0-t1

        set t0 = StopWatchMark(Clock)
        call SetTimerData(t,1)
        set t1 = StopWatchMark(Clock)
        set time[1] = t0-t1
            
        set t0 = StopWatchMark(Clock)
        set i = GetTimerData(t)
        set t1 = StopWatchMark(Clock)
        set time[2] = t0-t1
        
        set t0 = StopWatchMark(Clock)
        call ReleaseTimer(t)
        set t1 = StopWatchMark(Clock)
        set time[3] = t0-t1
            
        set t = null
        
        set t0 = StopWatchMark(Clock)
        set t = CreateTimer()
        set t1 = StopWatchMark(Clock)
        set time[4] = t0-t1

        set t0 = StopWatchMark(Clock)
        set Data[t] = 1
        set t1 = StopWatchMark(Clock)
        set time[5] = t0-t1
        
        set t0 = StopWatchMark(Clock)
        set i = Data[t]
        set t1 = StopWatchMark(Clock)
        set time[6] = t0-t1
        
        set t0 = StopWatchMark(Clock)
        call Data.flush(t)
        set t1 = StopWatchMark(Clock)
        set time[7] = t0-t1
        
        set t0 = StopWatchMark(Clock)
        call DestroyTimer(t)
        set t1 = StopWatchMark(Clock)
        set time[8] = t0-t1

endfunction

    function TestTimerUtils takes nothing returns nothing
        call TimerStart(CreateTimer(),0.,true,function test)
    endfunction
    
    private function SetUp takes nothing returns nothing
        set Data = HandleTable.create()
        set Clock = StopWatchCreate()
    endfunction

endscope

And since this is a system that's not used in real maps, it's ot important how long the library is or whatever. I think if it gives you small advanataged it's okay, too.
If a user claims your system being slow or that you could do something faster differently;
Graphics are always more persuading.
And since the screen of Emmeasure contains necessary information you can trust the test more as if it would only display these values:

So this makes your code a little shorter and displays very nice, trustable graphics.
What's wrong about using this over unwrapped StopWatch marks?
Also, as I said, the real usefulness shows when you have to place the benchmark code directly into other libraries. Causing open threads is very probable then.
Example:

JASS:

library PseudoNice
    globals
        private string array Pseudo
    endglobals
    
    function GetPseudo takes integer i returns string
        return Pseudo[i]
    endfunction
    
    private function ModifyPseudo takes integer i, integer b returns nothing
        if GetRandomInt(0,1) == 0 then
            set Pseudo[i] = "0"+ GetPseudo(b)
        else
            set Pseudo[i] = "1"+ GetPseudo(b)
        endif
    endfunction
    
    // Fired when a unit is attacked
    function UnitAttackts takes nothing returns nothing
        call ModifyPseudo(GetUnitKills(GetTriggerUnit()),GetUnitKills(GetAttacker()))
    endfunction
endlibrary

This:

JASS:

library PseudoNice
    globals
        private string array Pseudo
    endglobals
    
    function GetPseudo takes integer i returns string
        call Test.Start("Pseudo","ModifyPseudo")
        set i = Pseudo[i]
        call Test.End()
        return i
    endfunction
    
    private function ModifyPseudo takes integer i, integer b returns nothing
        call Test.Start("Pseudo","ModifyPseudo")
        if GetRandomInt(0,1) == 0 then
            set Pseudo[i] = "0"+ GetPseudo(b)
        else
            set Pseudo[i] = "1"+ GetPseudo(b)
        endif
        call Test.End()
    endfunction
    
    // Fired when a unit is attacked
    function UnitAttackts takes nothing returns nothing
        call Test.Start("Pseudo","UnitAttacks")
        call ModifyPseudo(GetUnitKills(GetTriggerUnit()),GetUnitKills(GetAttacker()))
        call Test.End()
    endfunction
endlibrary

Will be buggy and inaccurate.
Benchmark abouts the benchmark and gives you error messages. Normal StopWatch wont. And believe me, that is useful. I already caused many open threads when testing systems internally I wouldn't have detected otherwise because the systems weren't made by me and I didn't know them.
A perfect example is KT2:

http://www.hiveworkshop.com/forums/g...rs-2-a-123471/

which does many internal things and was hard to test. Without the benchmark library I wouldn't have been able to make accurate tests.
That's the thread where I started to think about the design and where I started making the systems. You should notice the many tests I did. Those tests would have seemed less important with pure values.

I am thinking about some possibilites now:
- I keep the code as it is (maybe make it shorter/remove some unneccesary uberfeatures)
and everythings fine.
- I make my own thread where users can request benchmarks or where I simply test many important things and attach all the screennshots to the first post (hashtables vs. handleId-offset vs UnitUserData for example)
- I offer two versions of Benchmark. The current (reworked a bit) and this one:

JASS:

library Emmeasure initializer Init
    globals
        private real array Time
private integer array Runs
        private real t0
        private real t1
        private integer Clock
        private integer System
        private boolean Running = false
        private integer sysc = 0
        private string array sysn
        private integer array sysm
        private string array StableWord
    endglobals

function Int2Hex takes integer int returns string
    local string charMap = "0123456789ABCDEF"
    local string hex = ""
    local integer index = 0
    if ((int < 0) or (int > 0x7FFFFFFF)) then
return "|cffff0000Invalid argument in function Int2Hex()!|r"
    endif
    loop
        set index = ModuloInteger(int, 0x10) + 1
        set int = int / 0x10
        set hex = SubStringBJ(charMap, index, index) + hex
        exitwhen (int == 0)
    endloop
    return hex
endfunction


function Per2Clr takes string text, integer r, integer g, integer b, integer alpha returns string
    local string array str
    if ((r >= 0) and (r <= 100) and (g >= 0) and (g <= 100) and (b >= 0) and (b <= 100) and (alpha >= 0) and (alpha <= 100)) then
        set r = R2I(0xFF / 100.000 * r)
        set g = R2I(0xFF / 100.000 * g)
        set b = R2I(0xFF / 100.000 * b)
        set alpha = R2I(0xFF / 100.000 * alpha)
        if (alpha <= 0xF) then
            set str[0] = "0" + Int2Hex(alpha)
        else
            set str[0] = Int2Hex(alpha)
        endif
        if (r <= 0xF) then
            set str[1] = "0" + Int2Hex(r)
        else
            set str[1] = Int2Hex(r)
        endif
        if (g <= 0xF) then
            set str[2] = "0" + Int2Hex(g)
        else
            set str[2] = Int2Hex(g)
        endif
        if (b <= 0xF) then
            set str[3] = "0" + Int2Hex(b)
        else
            set str[3] = Int2Hex(b)
        endif
        return "|c" + str[0] + str[1] + str[2] + str[3] + text
    else
        return "|cffff0000Invalid arguments in function Per2Clr()!|r"
    endif
endfunction

private function CreatePercentageBar takes real Percent returns string
    local string ret = Per2Clr("",100-R2I(Percent),R2I(Percent),0,0)
    local integer i = 0
    local boolean b = true
    loop
        set i = i + 1
        exitwhen i == 100
        if i-1 == R2I(Percent) and b then
            set ret = ret + "|cff808080"
            set b = false
        endif
        set ret = ret + "|"
    endloop
    return ret
endfunction

struct Test
    static method Declare takes string name, integer Max returns nothing
        set sysc = sysc + 1
        set sysn[sysc] = name
        set sysm[sysc] = Max
    endmethod

    static method Convert takes string name returns integer
        local integer i = 0
        loop
            set i = i + 1
            exitwhen i > sysc
            if name == sysn[i] then
                return i
            endif
        endloop
        call BJDebugMsg("TEST: System "+name+" is not registered.")
        return -1
    endmethod

    static method Start takes string Name returns nothing
        if Running == false then
            set Running = true
            set System = Test.Convert(Name)
            set Runs[System] = Runs[System] + 1
            set t0 = StopWatchMark(Clock)
        else
            call BJDebugMsg("TEST: "+sysn[System]+" is causing an open thread.")
        endif
    endmethod

    static method End takes string s returns nothing
        set t1 = StopWatchMark(Clock)
        set Time[System] = Time[System] + (t1-t0)
        set System = 0
        set Running = false
    endmethod

    static method Display takes string Name returns nothing
        local integer sys = Test.Convert(Name)
        call BJDebugMsg(Name+ " took "+R2S(Time[sys])+ " seconds for "+I2S(Runs[sys])+ " executions." )
    endmethod

    static method Compare takes nothing returns nothing
        local integer i = 0
        local real highest = Time[1]
        local real perc
        local real Exec = 0.
        local integer Temp
        loop
            set i = i + 1
            exitwhen i > sysc
            if Time[i] < highest then
                set highest = Time[i]
            endif
        endloop

        set i = 0
        call BJDebugMsg("================================")
        loop
            set i = i + 1
            exitwhen i > sysc
            set Exec = Exec + Runs[i]
            set perc = highest/Time[i]*100.
            call BJDebugMsg(sysn[i]+": "+R2S(perc)+"%")
            call BJDebugMsg(CreatePercentageBar(perc))
        endloop
        set Exec = Exec / sysc
        set Temp = R2I(Exec/150)
        if Temp > 7 then
            set Temp = 7
        endif
        call BJDebugMsg("Testing Quality: "+StableWord[Temp])
    endmethod
endstruct

private function Init takes nothing returns nothing
    set Clock = StopWatchCreate()
    set StableWord[0] = Per2Clr("unacceptable",100,0,0,0)
    set StableWord[1] = Per2Clr("poor",86,14,0,0)
    set StableWord[2] = Per2Clr("lacking",71,28,0,0)
    set StableWord[3] = Per2Clr("not bad",50,50,0,0)
    set StableWord[4] = Per2Clr("fine",30,70,0,0)
    set StableWord[5] = Per2Clr("safe",20,80,0,0)
    set StableWord[6] = Per2Clr("Great",13,87,0,0)
    set StableWord[7] = Per2Clr("Excellent",0,100,0,0)
endfunction
endlibrary

as "small version" of benchmark.

edit: I added a testmap with a template now for everybody to do his own benchmarks.
I use exactly the same testmap.