planners = ['agap', 'badff', 'badffp', 'cedalion', 'eroller', 'libacop', 'libacop2', 'llama', 'miplan', 'rollent', 'smac'] problems = ['elevators-p24_54_1', 'elevators-p32_40_1', 'elevators-p32_54_1', 'elevators-p40_42_1', 'elevators-p40_48_1', 'floortile-p4-3-2', 'floortile-p4-3-3', 'floortile-p4-5-3', 'floortile-p5-4-2', 'floortile-p6-4-3', 'nomystery-p-c11-09-07', 'nomystery-p-c15-10-08', 'nomystery-p-c15-11-11', 'nomystery-p-c15-12-10', 'nomystery-p-c15-15-14', 'parking-p34-28', 'parking-p36-19', 'parking-p44-23', 'parking-p46-24', 'parking-p58-30', 'spanner-prob-130-3-49..', 'spanner-prob-132-135-85..', 'spanner-prob-143-145-93..', 'spanner-prob-150-150-103..', 'spanner-prob-159-155-106..', 'transport-p-2-55-23', 'transport-p-3-55-31', 'transport-p-3-56-35', 'transport-p-3-58-27', 'transport-p-3-60-27'] Planner Quality NO Knowledge eroller 1.00 - - - - - - - - - - - - - - 0.02 - - - - - 0.04 - - - - - - - - 1.06 rollent 1.00 - - - - - - - - - - - - - - 0.02 - - - - - 0.04 - - - - - - - - 1.06 agap 1.00 - 0.98 - - - - - - - - 1.00 - - - - - - 1.00 1.00 1.00 - - - - 1.00 - 1.00 - - 7.98 cedalion 1.00 - - 0.84 - 1.00 - - - - 1.00 - - - 1.00 1.00 - - 0.50 0.44 - - - - - - 0.74 - 0.58 - 8.12 smac 1.00 - - 0.84 - 1.00 - - - - 1.00 - - - 1.00 1.00 - - 0.50 0.44 - - - - - - 0.74 - 0.58 - 8.12 badff 1.00 1.00 - - - - - 1.00 0.96 - 1.00 - - - - - 1.00 1.00 0.84 0.84 - 1.00 0.88 - - - - - 0.93 - 11.44 badffp 1.00 1.00 - - - - - 1.00 0.96 - 1.00 - - - - - 1.00 1.00 0.84 0.84 - 1.00 0.88 - - - - - 0.93 - 11.44 llama 1.00 0.98 - 1.00 - 0.85 - 0.88 1.00 - 1.00 - - - 0.75 0.87 - 0.72 1.00 0.84 - 0.62 0.97 1.00 1.00 - 1.00 - 1.00 - 16.47 libacop 1.00 - 0.83 0.51 - 0.57 1.00 - 0.89 - 1.00 - 1.00 - 0.36 0.53 0.56 0.50 1.00 1.00 0.98 - 1.00 0.97 0.86 0.94 0.49 - 0.68 - 16.66 libacop2 1.00 - 0.97 0.53 - 0.59 1.00 - 0.89 - 1.00 - 1.00 - 0.36 0.53 0.56 0.50 1.00 1.00 0.98 - 1.00 0.97 0.86 0.98 0.49 - 0.68 - 16.88 miplan 1.00 0.86 1.00 0.72 - 0.67 - 0.71 0.88 - 1.00 1.00 1.00 - 0.53 0.91 - - 1.00 1.00 1.00 0.81 1.00 0.95 0.91 1.00 0.58 1.00 0.68 - 20.22 Planner Quality With Knowledge badffp 0.31 0.69 - - 1.00 - - 0.84 - - 0.92 1.00 - - - - - - 0.84 0.63 - 0.59 0.78 0.73 0.86 - - 0.99 - - 10.17 badff 1.00 - - - - - - 0.94 0.96 - 1.00 - - - - - - 1.00 0.84 0.77 - 1.00 0.85 - - - - - 0.89 1.00 10.24 rollent 0.61 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - 1.00 - - - - 0.25 0.22 0.27 0.36 0.23 - 0.97 0.90 - 0.84 - 0.20 1.00 0.61 0.88 11.83 eroller 0.61 0.92 0.36 - 0.90 - - 0.62 - - 1.00 1.00 - - - 0.25 - - 0.34 0.30 0.32 0.92 0.97 0.93 0.94 - 0.20 0.99 - 0.97 12.51 llama 1.00 0.65 - 0.45 - 0.45 - 0.82 1.00 - 1.00 - - - 0.49 0.47 - 0.72 1.00 0.84 - 0.58 0.93 0.80 0.82 - 0.60 - 0.96 0.71 14.30 libacop 1.00 - 0.59 0.38 - 0.53 0.67 - 0.89 - 1.00 - - - 0.65 0.54 0.66 0.50 0.56 1.00 0.98 - 0.93 0.93 0.84 0.68 0.33 1.00 0.65 - 15.31 agap 1.00 0.71 1.00 - - - 0.73 1.00 0.96 - 1.00 1.00 - - - - 0.69 0.48 1.00 1.00 1.00 0.96 0.82 - - 1.00 - 1.00 0.59 - 15.94 libacop2 1.00 - 0.59 0.38 - 0.53 0.67 - 0.89 - 1.00 - 0.98 - 0.32 0.61 0.66 0.50 0.56 1.00 0.98 0.90 0.93 0.93 0.84 0.68 - 1.00 0.65 - 16.60 smac 1.00 0.67 0.94 0.27 - 0.33 0.50 0.77 0.60 - 1.00 - 1.00 - 0.30 0.34 0.40 0.37 0.97 1.00 0.91 0.74 1.00 1.00 1.00 0.94 0.29 - 0.37 0.73 17.45 cedalion 1.00 - 0.98 1.00 - 1.00 1.00 0.91 0.73 - 1.00 - 1.00 - 1.00 1.00 0.69 0.69 1.00 1.00 0.98 - 1.00 1.00 1.00 - 1.00 - 1.00 - 19.98 miplan 1.00 0.59 0.91 0.61 - 0.55 0.65 0.72 0.99 - 1.00 1.00 1.00 - 0.53 0.56 1.00 0.96 1.00 1.00 1.00 0.75 0.97 0.95 0.97 0.98 0.55 1.00 0.65 - 21.88 Core Solvers Quality NO Knowledge eroller 1.00 - - - - - - - - - - - - - - 0.02 - - - - - 0.04 - - - - - - - - 1.06 rollent 1.00 - - - - - - - - - - - - - - 0.02 - - - - - 0.04 - - - - - - - - 1.06 smac 1.00 - - 0.84 - 1.00 - - - - 1.00 - - - 1.00 1.00 - - 0.50 0.44 - - - - - - 0.74 - 0.58 - 8.12 badff 1.00 1.00 - - - - - 1.00 0.96 - 1.00 - - - - - 1.00 1.00 0.84 0.84 - 1.00 0.88 - - - - - 0.93 - 11.44 llama 1.00 0.98 - 1.00 - 0.85 - 0.88 1.00 - 1.00 - - - 0.75 0.87 - 0.72 1.00 0.84 - 0.62 0.97 1.00 1.00 - 1.00 - 1.00 - 16.47 Core Solvers Quality With Knowledge badff 1.00 - - - - - - 0.94 0.96 - 1.00 - - - - - - 1.00 0.84 0.77 - 1.00 0.85 - - - - - 0.89 1.00 10.24 rollent 0.61 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - 1.00 - - - - 0.25 0.22 0.27 0.36 0.23 - 0.97 0.90 - 0.84 - 0.20 1.00 0.61 0.88 11.83 eroller 0.61 0.92 0.36 - 0.90 - - 0.62 - - 1.00 1.00 - - - 0.25 - - 0.34 0.30 0.32 0.92 0.97 0.93 0.94 - 0.20 0.99 - 0.97 12.51 llama 1.00 0.65 - 0.45 - 0.45 - 0.82 1.00 - 1.00 - - - 0.49 0.47 - 0.72 1.00 0.84 - 0.58 0.93 0.80 0.82 - 0.60 - 0.96 0.71 14.30 smac 1.00 0.67 0.94 0.27 - 0.33 0.50 0.77 0.60 - 1.00 - 1.00 - 0.30 0.34 0.40 0.37 0.97 1.00 0.91 0.74 1.00 1.00 1.00 0.94 0.29 - 0.37 0.73 17.45 Planner Quality Ordered by Delta Knowledge llama 0.00 -0.32 - -0.55 - -0.40 - -0.05 0.00 - 0.00 - - - -0.26 -0.40 - 0.00 0.00 0.00 - -0.04 -0.03 -0.20 -0.18 - -0.40 - -0.04 0.71 -2.18 libacop 0.00 - -0.24 -0.13 - -0.05 -0.33 - 0.00 - 0.00 - - - 0.30 0.01 0.11 0.00 -0.44 0.00 0.00 - -0.07 -0.05 -0.02 -0.25 -0.16 1.00 -0.03 - -0.35 libacop2 0.00 - -0.38 -0.15 - -0.06 -0.33 - 0.00 - 0.00 - -0.02 - -0.04 0.09 0.11 0.00 -0.44 0.00 0.00 0.90 -0.07 -0.05 -0.02 -0.30 - 1.00 -0.03 - 0.21 badff 0.00 - - - - - - -0.06 0.00 - 0.00 - - - - - - 0.00 0.00 -0.06 - 0.00 -0.03 - - - - - -0.04 1.00 0.81 miplan 0.00 -0.27 -0.09 -0.11 - -0.12 0.65 0.01 0.11 - 0.00 0.00 0.00 - 0.00 -0.35 1.00 0.96 0.00 0.00 0.00 -0.06 -0.03 -0.00 0.06 -0.02 -0.03 0.00 -0.03 - 1.66 badffp -0.69 -0.31 - - 1.00 - - -0.16 - - -0.08 1.00 - - - - - - 0.00 -0.21 - -0.41 -0.10 0.73 0.86 - - 0.99 - - 2.62 agap 0.00 0.71 0.02 - - - 0.73 1.00 0.96 - 1.00 0.00 - - - - 0.69 0.48 0.00 0.00 0.00 0.96 0.82 - - 0.00 - 0.00 0.59 - 7.95 smac 0.00 0.67 0.94 -0.57 - -0.67 0.50 0.77 0.60 - 0.00 - 1.00 - -0.70 -0.66 0.40 0.37 0.46 0.56 0.91 0.74 1.00 1.00 1.00 0.94 -0.45 - -0.21 0.73 9.33 rollent -0.39 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - 1.00 - - - - 0.23 0.22 0.27 0.36 0.23 - 0.93 0.90 - 0.84 - 0.20 1.00 0.61 0.88 10.77 eroller -0.39 0.92 0.36 - 0.90 - - 0.62 - - 1.00 1.00 - - - 0.23 - - 0.34 0.30 0.32 0.87 0.97 0.93 0.94 - 0.20 0.99 - 0.97 11.45 cedalion 0.00 - 0.98 0.16 - 0.00 1.00 0.91 0.73 - 0.00 - 1.00 - 0.00 0.00 0.69 0.69 0.50 0.56 0.98 - 1.00 1.00 1.00 - 0.26 - 0.42 - 11.86 Core Solvers Quality Ordered by Delta Knowledge llama 0.00 -0.32 - -0.55 - -0.40 - -0.05 0.00 - 0.00 - - - -0.26 -0.40 - 0.00 0.00 0.00 - -0.04 -0.03 -0.20 -0.18 - -0.40 - -0.04 0.71 -2.18 badff 0.00 - - - - - - -0.06 0.00 - 0.00 - - - - - - 0.00 0.00 -0.06 - 0.00 -0.03 - - - - - -0.04 1.00 0.81 smac 0.00 0.67 0.94 -0.57 - -0.67 0.50 0.77 0.60 - 0.00 - 1.00 - -0.70 -0.66 0.40 0.37 0.46 0.56 0.91 0.74 1.00 1.00 1.00 0.94 -0.45 - -0.21 0.73 9.33 rollent -0.39 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - 1.00 - - - - 0.23 0.22 0.27 0.36 0.23 - 0.93 0.90 - 0.84 - 0.20 1.00 0.61 0.88 10.77 eroller -0.39 0.92 0.36 - 0.90 - - 0.62 - - 1.00 1.00 - - - 0.23 - - 0.34 0.30 0.32 0.87 0.97 0.93 0.94 - 0.20 0.99 - 0.97 11.45 Problem spanner-prob-130-3-49.. was solved by at least 7 planners without knowledge and is too easy Problem nomystery-p-c11-09-07 was solved by at least 7 planners without knowledge and is too easy Problem elevators-p32_40_1 was solved by at least 7 planners without knowledge and is too easy Problem floortile-p4-3-2 was solved by at least 7 planners without knowledge and is too easy Problem floortile-p4-3-3 was solved by at least 7 planners without knowledge and is too easy Problem transport-p-3-60-27 was solved by at least 7 planners without knowledge and is too easy Planner Quality on Harder Problems Ordered by Delta Knowledge llama -0.32 - -0.55 - -0.40 - -0.05 0.00 - - - - -0.26 - 0.00 - -0.04 -0.03 -0.20 -0.18 - -0.40 - 0.71 -1.73 libacop - -0.24 -0.13 - -0.05 -0.33 - 0.00 - - - - 0.30 0.11 0.00 0.00 - -0.07 -0.05 -0.02 -0.25 -0.16 1.00 - 0.10 libacop2 - -0.38 -0.15 - -0.06 -0.33 - 0.00 - - -0.02 - -0.04 0.11 0.00 0.00 0.90 -0.07 -0.05 -0.02 -0.30 - 1.00 - 0.59 badff - - - - - - -0.06 0.00 - - - - - - 0.00 - 0.00 -0.03 - - - - - 1.00 0.91 miplan -0.27 -0.09 -0.11 - -0.12 0.65 0.01 0.11 - 0.00 0.00 - 0.00 1.00 0.96 0.00 -0.06 -0.03 -0.00 0.06 -0.02 -0.03 0.00 - 2.04 badffp -0.31 - - 1.00 - - -0.16 - - 1.00 - - - - - - -0.41 -0.10 0.73 0.86 - - 0.99 - 3.60 agap 0.71 0.02 - - - 0.73 1.00 0.96 - 0.00 - - - 0.69 0.48 0.00 0.96 0.82 - - 0.00 - 0.00 - 6.37 rollent 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - - - - - 0.22 0.27 - 0.93 0.90 - 0.84 - 0.20 1.00 0.88 8.74 smac 0.67 0.94 -0.57 - -0.67 0.50 0.77 0.60 - - 1.00 - -0.70 0.40 0.37 0.91 0.74 1.00 1.00 1.00 0.94 -0.45 - 0.73 9.18 eroller 0.92 0.36 - 0.90 - - 0.62 - - 1.00 - - - - - 0.32 0.87 0.97 0.93 0.94 - 0.20 0.99 0.97 9.97 cedalion - 0.98 0.16 - 0.00 1.00 0.91 0.73 - - 1.00 - 0.00 0.69 0.69 0.98 - 1.00 1.00 1.00 - 0.26 - - 10.40 Core Solver Quality on Harder Problems Ordered by Delta Knowledge llama -0.32 - -0.55 - -0.40 - -0.05 0.00 - - - - -0.26 - 0.00 - -0.04 -0.03 -0.20 -0.18 - -0.40 - 0.71 -1.73 badff - - - - - - -0.06 0.00 - - - - - - 0.00 - 0.00 -0.03 - - - - - 1.00 0.91 rollent 1.00 0.20 - 0.95 - 0.33 0.78 0.25 - - - - - 0.22 0.27 - 0.93 0.90 - 0.84 - 0.20 1.00 0.88 8.74 smac 0.67 0.94 -0.57 - -0.67 0.50 0.77 0.60 - - 1.00 - -0.70 0.40 0.37 0.91 0.74 1.00 1.00 1.00 0.94 -0.45 - 0.73 9.18 eroller 0.92 0.36 - 0.90 - - 0.62 - - 1.00 - - - - - 0.32 0.87 0.97 0.93 0.94 - 0.20 0.99 0.97 9.97